Skip to content

Instantly share code, notes, and snippets.

@josericardo
Last active September 26, 2023 06:21
Show Gist options
  • Save josericardo/5102304 to your computer and use it in GitHub Desktop.
Save josericardo/5102304 to your computer and use it in GitHub Desktop.
Helper to iterate over pageable sources. Should reduce memory usage when querying large tables via Spring Data.
======================================
Usage:
Fetcher<Source, MyEntity> f = new Fetcher<Source, MyEntity>(source) {
@Override
public List<MyEntity> fetch(Pageable pageRequest)
{
return source.findAll(pageRequest);
}
};
PageableCollection<Source,MyEntity> pageableCollection = new PageableCollection<Source, MyEntity>(f);
for (MyEntity e : pageableCollection) {
// ...
}
============================================================================
public class PageableCollection<T> implements Iterable<T>
{
private static final int DEFAULT_PAGE_SIZE = 100;
private Fetcher<?, T> fetcher;
private int pageSize;
public PageableCollection(Fetcher<?, T> f) {
this(f, DEFAULT_PAGE_SIZE);
}
public PageableCollection(Fetcher<?, T> f, int pageSize)
{
this.fetcher = f;
this.pageSize = pageSize;
}
@Override
public Iterator<T> iterator()
{
return new PageableIterator<T>(fetcher, pageSize);
}
}
/**
* Initially makes sense only inside the PageableCollection class
*/
class PageableIterator<T> implements Iterator<T>
{
private static final int FIRST_PAGE = 0;
private List<T> currentData;
private int cursor;
private int pageSize;
private Pageable page;
private Fetcher<?, T> fetcher;
public PageableIterator(Fetcher<?, T> f, int pageSize) {
this.fetcher = f;
this.pageSize = pageSize;
page = new PageRequest(FIRST_PAGE, pageSize);
currentData = new ArrayList<T>();
}
@Override
public boolean hasNext()
{
if (hasDataLoaded()) {
return true;
}
tryToFetchMoreData();
return !currentData.isEmpty();
}
private void tryToFetchMoreData()
{
currentData = fetcher.fetch(page);
page = new PageRequest(page.getPageNumber()+1, pageSize);
cursor = 0;
}
private boolean hasDataLoaded()
{
return cursor < currentData.size();
}
@Override
public T next()
{
return currentData.get(cursor++);
}
@Override
public void remove()
{
throw new UnsupportedOperationException();
}
}
public abstract class Fetcher<S, T>
{
protected S source;
public Fetcher(S s) {
this.source = s;
}
public abstract List<T> fetch(Pageable pageRequest);
}
@axiopisty
Copy link

Just FYI, this implementation is not thread safe. Appropriate synchronization should be used when modifying the currentData, page, etc.. members. Also, in the tryToFetchMoreData method, it might be better to use Pageable's next method rather than creating a new PageRequest(page.getPageNumber()+1, pageSize) in case the elements in the database have been modified during the process of iteration.

@NielsHaar
Copy link

Nice! Thanks.
@axiopisty Also the sort information won't get lost if present.

@jahdasa
Copy link

jahdasa commented Jan 18, 2023

Thanks a lot,
I just refactored it to make usage more readable and general:

=============================================================
Usage:

    final Fetcher<Portfolio> fetcher = (pageable) ->
              portfolioRepository.getAllByAssetType(AssetType.PORTFOLIO, pageable);

    for (final Portfolio portfolio : fetcher.toPageableCollection())
    {
        System.out.println(portfolio);
    }

=============================================================

    @FunctionalInterface
    public interface Fetcher<T>
    {
        List<T> fetch(Pageable pageRequest);
    
        default PageableCollection<T> toPageableCollection()
        {
            return new PageableCollection<>(this);
        }
    }

=============================================================

    public class PageableCollection<T> implements Iterable<T>
    {
        private static final int DEFAULT_PAGE_SIZE = 1000;
    
        private final Fetcher<T> fetcher;
        private final int pageSize;
    
        public PageableCollection(final Fetcher<T> fetcher)
        {
            this(fetcher, DEFAULT_PAGE_SIZE);
        }
    
        public PageableCollection(final Fetcher<T> fetcher, final int pageSize)
        {
            this.fetcher = fetcher;
            this.pageSize = pageSize;
        }
    
        @Override
        public Iterator<T> iterator()
        {
            return new PageableIterator<>(fetcher, pageSize);
        }
    }

=============================================================

    @Slf4j
    class PageableIterator<T> implements Iterator<T>
    {
        private static final int FIRST_PAGE = 0;
    
        private final Fetcher<T> fetcher;
        private final int pageSize;
    
        private List<T> currentData;
        private int cursor;
        private Pageable page;
    
        public PageableIterator(final Fetcher<T> fetcher, final int pageSize)
        {
            this.fetcher = fetcher;
            this.pageSize = pageSize;
            page = PageRequest.of(FIRST_PAGE, pageSize);
            currentData = new ArrayList<>();
        }
    
        @Override
        public boolean hasNext()
        {
            if (hasDataLoaded())
            {
                return true;
            }
    
            tryToFetchMoreData();
            return !currentData.isEmpty();
        }
    
        private void tryToFetchMoreData()
        {
            log.debug("try to fetch more data on page {}", page);
            currentData = fetcher.fetch(page);
            page = PageRequest.of(page.getPageNumber() + 1, pageSize);
            cursor = 0;
        }
    
        private boolean hasDataLoaded()
        {
            return cursor < currentData.size();
        }
    
        @Override
        public T next()
        {
            return currentData.get(cursor++);
        }
    
        @Override
        public void remove()
        {
            throw new UnsupportedOperationException();
        }
    }

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment