Helper to iterate over pageable sources. Should reduce memory usage when querying large tables via Spring Data.

PageableCollection.java

======================================
Usage:

Fetcher<Source, MyEntity> f = new Fetcher<Source, MyEntity>(source) {
	@Override
	public List<MyEntity> fetch(Pageable pageRequest)
	{
		return source.findAll(pageRequest);
	}
};

PageableCollection<Source,MyEntity> pageableCollection = new PageableCollection<Source, MyEntity>(f);

for (MyEntity e : pageableCollection) {
	// ...
}

============================================================================

public class PageableCollection<T> implements Iterable<T>
{
	private static final int DEFAULT_PAGE_SIZE = 100;
	
	private Fetcher<?, T> fetcher;
	private int pageSize;

	public PageableCollection(Fetcher<?, T> f) {
		this(f, DEFAULT_PAGE_SIZE);
	}
	
	public PageableCollection(Fetcher<?, T> f, int pageSize)
	{
		this.fetcher = f;
		this.pageSize = pageSize;
	}

	@Override
	public Iterator<T> iterator()
	{
		return new PageableIterator<T>(fetcher, pageSize);
	}
}

/**
 * Initially makes sense only inside the PageableCollection class
 */
class PageableIterator<T> implements Iterator<T>
{
	private static final int FIRST_PAGE = 0;
	
	private List<T> currentData;
	private int cursor;
	private int pageSize;
	private Pageable page;
	private Fetcher<?, T> fetcher;
	
	public PageableIterator(Fetcher<?, T> f, int pageSize) {
		this.fetcher = f;
		this.pageSize = pageSize;
		page = new PageRequest(FIRST_PAGE, pageSize);
		currentData = new ArrayList<T>();
	}
	
	@Override
	public boolean hasNext()
	{
		if (hasDataLoaded()) {
			return true;
		}
		
		tryToFetchMoreData();
		return !currentData.isEmpty();
	}
	
	private void tryToFetchMoreData()
	{
		currentData = fetcher.fetch(page);
		page = new PageRequest(page.getPageNumber()+1, pageSize);
		cursor = 0;
	}
	
	private boolean hasDataLoaded()
	{
		return cursor < currentData.size();
	}
	
	@Override
	public T next()
	{
		return currentData.get(cursor++);
	}
	
	@Override
	public void remove()
	{
		throw new UnsupportedOperationException();
	}
}

public abstract class Fetcher<S, T>
{
  protected S source;
	
	public Fetcher(S s) {
		this.source = s;
	}
	
	public abstract List<T> fetch(Pageable pageRequest);
}

Thanks a lot!

Just FYI, this implementation is not thread safe. Appropriate synchronization should be used when modifying the currentData, page, etc.. members. Also, in the tryToFetchMoreData method, it might be better to use Pageable's next method rather than creating a new PageRequest(page.getPageNumber()+1, pageSize) in case the elements in the database have been modified during the process of iteration.

Nice! Thanks.
@axiopisty Also the sort information won't get lost if present.

Thanks a lot,
I just refactored it to make usage more readable and general:

=============================================================
Usage:

    final Fetcher<Portfolio> fetcher = (pageable) ->
              portfolioRepository.getAllByAssetType(AssetType.PORTFOLIO, pageable);

    for (final Portfolio portfolio : fetcher.toPageableCollection())
    {
        System.out.println(portfolio);
    }

=============================================================

    @FunctionalInterface
    public interface Fetcher<T>
    {
        List<T> fetch(Pageable pageRequest);
    
        default PageableCollection<T> toPageableCollection()
        {
            return new PageableCollection<>(this);
        }
    }

=============================================================

    public class PageableCollection<T> implements Iterable<T>
    {
        private static final int DEFAULT_PAGE_SIZE = 1000;
    
        private final Fetcher<T> fetcher;
        private final int pageSize;
    
        public PageableCollection(final Fetcher<T> fetcher)
        {
            this(fetcher, DEFAULT_PAGE_SIZE);
        }
    
        public PageableCollection(final Fetcher<T> fetcher, final int pageSize)
        {
            this.fetcher = fetcher;
            this.pageSize = pageSize;
        }
    
        @Override
        public Iterator<T> iterator()
        {
            return new PageableIterator<>(fetcher, pageSize);
        }
    }

=============================================================

    @Slf4j
    class PageableIterator<T> implements Iterator<T>
    {
        private static final int FIRST_PAGE = 0;
    
        private final Fetcher<T> fetcher;
        private final int pageSize;
    
        private List<T> currentData;
        private int cursor;
        private Pageable page;
    
        public PageableIterator(final Fetcher<T> fetcher, final int pageSize)
        {
            this.fetcher = fetcher;
            this.pageSize = pageSize;
            page = PageRequest.of(FIRST_PAGE, pageSize);
            currentData = new ArrayList<>();
        }
    
        @Override
        public boolean hasNext()
        {
            if (hasDataLoaded())
            {
                return true;
            }
    
            tryToFetchMoreData();
            return !currentData.isEmpty();
        }
    
        private void tryToFetchMoreData()
        {
            log.debug("try to fetch more data on page {}", page);
            currentData = fetcher.fetch(page);
            page = PageRequest.of(page.getPageNumber() + 1, pageSize);
            cursor = 0;
        }
    
        private boolean hasDataLoaded()
        {
            return cursor < currentData.size();
        }
    
        @Override
        public T next()
        {
            return currentData.get(cursor++);
        }
    
        @Override
        public void remove()
        {
            throw new UnsupportedOperationException();
        }
    }