Specialized space partitioning container to solve a specific problem. Just for my own future reference. Sorry, probably not very useful for others due to the dependencies.

AcceleratedSpacePartition.cs

/// <summary>
/// Simple specialized container for handling large amounts of linearly moving objects in large spaces.
/// Can be used when an Octree or a BVH is too expensive to maintain, if there is a known area of space where AABB queries are concentrated.
/// Partitions space by creating an accelerated zone around a target object where AABB queries are faster to execute.
/// Objects outside the accelerated zone are throttled by feeding them aggregate delta times to drastically reduce the workload of updating them.
/// - Returns accurate results for queries outside the zone as well, but at a cost of brute force query + overhead.
/// - Throttling is not load-balanced over multiple ticks, i.e. every X tick is more expensive to execute.
/// </summary>
public class AcceleratedSpacePartition
{
    // Default settings
    protected static readonly Config _defaultConfig = new Config()
    {
        DistantObjectThrottling = 6,
        GrowZoneToLargestQuery = true,
        ZoneExtentX = 50,
        ZoneExtentY = 50,
        ZoneExtentZ = 50,
    };

    // Instance settings
    protected bool _canGrowZone;
    protected int _distantObjectThrottling;
    protected Vector3 _acceleratedZoneExtentMin;
    protected Vector3 _acceleratedZoneExtentMax;

    // Containers & queues
    protected readonly HashList<WorldObject> _distantObjects = new HashList<WorldObject>(1000);
    protected readonly HashList<WorldObject> _acceleratedObjects = new HashList<WorldObject>(200);
    protected readonly Queue<WorldObject> _removalQueue = new Queue<WorldObject>(10);
    protected readonly Queue<WorldObject> _additionQueue = new Queue<WorldObject>(10);

    // State
    protected Type _trackedType;
    protected WorldObject _trackedObject = null;
    protected BoundingBox _acceleratedZoneBox;
    protected int _tickCount = 0;
    protected TimeSpan _aggregateDeltaTime = TimeSpan.Zero;
    protected bool _updateRunning;

    /// <summary>
    /// Initializes the class with T as the type to build the accelerated zone around.
    /// </summary>
    /// <typeparam name="T">The type of the object that creates an accelerated zone around it. This should be where bounding box queries will take place, e.g. Camera, Player.</typeparam>
    public void Initialize<T>(Config config = null)
    {
        _trackedType = typeof(T);

        config = config ?? _defaultConfig;
        _acceleratedZoneExtentMin = _acceleratedZoneExtentMax = new Vector3(config.ZoneExtentX, config.ZoneExtentY, config.ZoneExtentZ);
        _distantObjectThrottling = config.DistantObjectThrottling;
        _canGrowZone = config.GrowZoneToLargestQuery;
    }

    /// <summary>
    /// Adds an object.
    /// </summary>
    public void Add(WorldObject obj)
    {
        if (obj == null) return;

        // Postpone action if incurred during update
        if (_updateRunning)
        {
            _additionQueue.Enqueue(obj);
            return;
        }

        // Special case if added object is the tracked object
        if (_trackedType.IsAssignableFrom(obj.GetType()))
            InitAcceleratedZone(obj);

        // If new object intersects accelerated zone, add to it; otherwise add it to distant objects
        if (_trackedObject != null && _acceleratedZoneBox.Intersects(obj.BoundingBox))
            _acceleratedObjects.Add(obj);
        else
            _distantObjects.Add(obj);
    }

    /// <summary>
    /// Removes an object.
    /// </summary>
    public void Remove(WorldObject obj)
    {
        if (obj == null) return;

        // Postpone action if incurred during update
        if (_updateRunning)
        {
            _removalQueue.Enqueue(obj);
            return;
        }

        // Special case when removed object is the tracked object
        if (obj == _trackedObject)
            ResetAcceleratedZone();

        // Actual remove
        if (_acceleratedObjects.Contains(obj))
            _acceleratedObjects.Remove(obj);
        else
            _distantObjects.Remove(obj);
    }

    /// <summary>
    /// Checks if moved objects are still located in the zone they belong to, and if not, transfers them.
    /// </summary>
    public void OnObjectMoved(WorldObject obj, Vector3 displacement)
    {
        // If tracked object/zone isn't set up, we can't do anything.
        if (_trackedObject == null)
            return;

        // Technically here we should check if object is tracked object, and if so, return.
        // But there is no real harm, and excluding that single object from processing would incur type checking on lots of other objects.

        bool belongsToAcceleratedZone = _acceleratedZoneBox.Intersects(obj.BoundingBox);
        bool storedInAcceleratedZone = _acceleratedObjects.Contains(obj);

        // Action required case 1: Object needs to be moved from accelerated zone to distant
        if (!belongsToAcceleratedZone && storedInAcceleratedZone)
        {
            TransferObject(obj,
                from: _acceleratedObjects,
                to: _distantObjects);
            return;
        }

        // Action required case 2: Object needs to be moved from distant to accelerated zone
        if (belongsToAcceleratedZone && !storedInAcceleratedZone)
        {
            TransferObject(obj,
                from: _distantObjects,
                to: _acceleratedObjects);
            return;
        }
    }

    /// <summary>
    /// Execute bounding box query.
    /// </summary>
    public void Query(BoundingBox box, List<WorldObject> hits)
    {
        var containment = _acceleratedZoneBox.Contains(box);
        switch (containment)
        {
            // Fast path for boxes inside accelerated zone = Tests only accelerated objects
            case ContainmentType.Contains:
                QueryList(_acceleratedObjects, box, hits);
                break;

            // Slow path for boxes outside accelerated zone = Tests all objects, except accelerated objects
            // Interpreted as query that naturally lies outside of zone
            case ContainmentType.Disjoint:
                EnsureDistantUpdated(); // Push accumulated deltaTime to throttled objects to bring them out of stale state
                QueryList(_distantObjects, box, hits);
                break;

            // Slowest path for boxes partially outside accelerated zone = Tests ALL objects
            // Interpreted as query that failed to be contained
            case ContainmentType.Intersects:
                TryGrowZoneExtent(box); // Try prevent this from happening again
                EnsureDistantUpdated();
                QueryList(_acceleratedObjects, box, hits);
                QueryList(_distantObjects, box, hits);
                break;

            default:
                throw new Exception($"Unexpected enum value '{containment}'.");
        }

        // Execute query on a specific list
        void QueryList(HashList<WorldObject> list, BoundingBox mybox, List<WorldObject> myhits)
        {
            var count = list.Count;
            for (int i = 0; i < count; i++)
            {
                var obj = list[i];
                if (obj.BoundingBox.Intersects(mybox))
                    myhits.Add(obj);
            }
        }

        // This will hurt :)
        // TODO: Make sure it works as expected
        void EnsureDistantUpdated()
        {
            // Nothing to push (lucky)
            if (_aggregateDeltaTime == TimeSpan.Zero)
                return;

            _updateRunning = true;
            DoUpdateOn(_distantObjects, _aggregateDeltaTime);
            _updateRunning = false;
            ExecuteQueues(); // Clean up potential additions and removals postponed during update
            _aggregateDeltaTime = TimeSpan.Zero;
        }
    }

    /// <summary>
    /// Execute update on objects.
    /// </summary>
    public void Update(TimeSpan deltaTime)
    {
        { // Bookkeeping
            _updateRunning = true;
            _tickCount++;
            _aggregateDeltaTime += deltaTime;
            CreateAcceleratedZoneBox();
        }

        // Update accelerated objects - these are not throttled
        DoUpdateOn(_acceleratedObjects, deltaTime);

        // Update throttled distant objects
        if (_tickCount % _distantObjectThrottling == 0)
        {
            DoUpdateOn(_distantObjects, _aggregateDeltaTime);
            _aggregateDeltaTime = TimeSpan.Zero; // Consumed
        }

        { // Bookkeeping
            _updateRunning = false;
        }

        // Execute additions and removals incurred during update
        ExecuteQueues();
    }

    /// <summary>
    /// Important to call each tick.
    /// Updates the bounding box of the accelerated zone based on the current position of the tracked object.
    /// </summary>
    protected void CreateAcceleratedZoneBox()
    {
        if (_trackedObject == null)
            return;
        _acceleratedZoneBox = new BoundingBox(_trackedObject.Position - _acceleratedZoneExtentMin, _trackedObject.Position + _acceleratedZoneExtentMax);
    }

    /// <summary>
    /// Executes updates on the given list.
    /// </summary>
    protected void DoUpdateOn(HashList<WorldObject> list, TimeSpan delta)
    {
        // Backwards iteration supports object removal from collection (i.e. transfer from distant to close)
        var count = list.Count;
        for (int i = count - 1; i >= 0; i--)
            list[i].Update(delta);
        count = list.Count; // New count needed due to potential transitions
        for (int i = count - 1; i >= 0; i--)
            list[i].PostUpdate();
    }

    /// <summary>
    /// Execute object addition and removal queues.
    /// </summary>
    protected void ExecuteQueues()
    {
        while (_removalQueue.Count > 0)
            Remove(_removalQueue.Dequeue());

        while (_additionQueue.Count > 0)
            Add(_additionQueue.Dequeue());
    }

    /// <summary>
    /// Removes all objects from the collection and resets run stats.
    /// </summary>
    public void Clear()
    {
        if (_updateRunning)
            throw new InvalidOperationException("Clearing is not supported while Update is running.");

        ResetAcceleratedZone();
        _distantObjects.Clear();
        _aggregateDeltaTime = TimeSpan.Zero;
        _tickCount = 0;
    }

    /// <summary>
    /// Helper method to transfer an object from one list to another.
    /// </summary>
    [MethodImpl(256)]
    protected void TransferObject(WorldObject obj, HashList<WorldObject> from, HashList<WorldObject> to)
    {
        from.Remove(obj);
        to.Add(obj);
    }

    /// <summary>
    /// Initializes accelerated zone settings with the given object.
    /// </summary>
    protected void InitAcceleratedZone(WorldObject trackedObject)
    {
        if (_trackedObject != null)
            throw new InvalidOperationException("Support for adding multiple tracked objects is not implemented.");

        if (!_trackedType.IsAssignableFrom(trackedObject.GetType()))
            throw new InvalidOperationException("Specified object is not a valid tracked object");

        _trackedObject = trackedObject;
        CreateAcceleratedZoneBox();

        // No need to recalculate, because objects will be transferred automatically after they moved.
    }

    /// <summary>
    /// Flushes accelerated zone and nulls tracked object.
    /// </summary>
    protected void ResetAcceleratedZone()
    {
        _distantObjects.AddRange(_acceleratedObjects);
        _acceleratedObjects.Clear();
        _trackedObject = null;
        _acceleratedZoneBox = new BoundingBox(Vector3.Zero, Vector3.Zero);
    }

    /// <summary>
    /// If allowed, grows the extent of the accelerated zone to encompass the given BoundingBox.
    /// </summary>
    protected void TryGrowZoneExtent(BoundingBox box)
    {
        if (!_canGrowZone)
            return;

        // If currently queried box is absurdly larger, skip growing
        if (box.Size.Sum / _acceleratedZoneBox.Size.Sum > 100) // TODO: Factor out hard-coded value
            return;

        var oldCenter = _acceleratedZoneBox.Center;
        _acceleratedZoneBox.Include(ref box);

        // Derive new extents from inflated box, because box is transient; it's updated every tick based on current position + extents
        _acceleratedZoneExtentMin = oldCenter - _acceleratedZoneBox.Min;  // TODO: Symmetrical growth can lead to overaly large extents
        _acceleratedZoneExtentMin *= 1.05f; // TODO: Factor out hard-coded imprecision margin value
        _acceleratedZoneExtentMax = _acceleratedZoneBox.Max - oldCenter;
        _acceleratedZoneExtentMax *= 1.05f;
    }

    /// <summary>
    /// Flushes and rebuilds the accelerated zone. No idea why would you want to call it.
    /// </summary>
    public void RecalculateAcceleratedZone()
    {
        if (_trackedObject == null)
            throw new InvalidOperationException("Recalculation requires a valid tracked object to be set.");

        // Flush
        _distantObjects.AddRange(_acceleratedObjects);
        _acceleratedObjects.Clear();

        // Search for tracked object and init if found
        foreach (var obj in _distantObjects)
            if (_trackedType.IsAssignableFrom(obj.GetType()))
                InitAcceleratedZone(obj);

        // Re-check all objects and transfer when needed
        var count = _distantObjects.Count;
        for (int i = count-1; i >= 0; i--)
        {
            var obj = _distantObjects[i];
            if (_acceleratedZoneBox.Intersects(obj.BoundingBox))
                TransferObject(obj,
                    from: _distantObjects,
                    to: _acceleratedObjects);
        }
    }

    public class Config
    {
        /// <summary>
        /// The X extent of accelerated zone. Smaller zone is faster, but queries outside the extent drastically reduce performance.
        /// </summary>
        public float ZoneExtentX;

        /// <summary>
        /// The Y extent of accelerated zone. Smaller zone is faster, but queries outside the extent drastically reduce performance.
        /// </summary>
        public float ZoneExtentY;

        /// <summary>
        /// The Z extent of accelerated zone. Smaller zone is faster, but queries outside the extent drastically reduce performance.
        /// </summary>
        public float ZoneExtentZ;

        /// <summary>
        /// Throttle distant object updates to every X tick.
        /// </summary>
        public int DistantObjectThrottling;

        /// <summary>
        /// Expand accelerated zone extents if zone-exceeding queries are detected.
        /// </summary>
        public bool GrowZoneToLargestQuery;
    }
}