A C# function that fetches all the items returned by a service across several pages

aws-s3.cs

IAmazonS3 client = new AmazonS3Client (RegionEndpoint.EUWest1);

var objects = await FetchAll<S3Object, ListObjectsV2Request, ListObjectsV2Response>(
					() => new ListObjectsV2Request { BucketName = "my-bucket", MaxKeys = 1000 },
					req => client.ListObjectsV2Async (req),
					res => res.NextContinuationToken,
					(req, token) => req.ContinuationToken = token,
					res => res.S3Objects,
					res => res.IsTruncated
);

objects.Dump();

aws-secretsmanager.cs

IAmazonSecretsManager client = new AmazonSecretsManagerClient (RegionEndpoint.EUWest1);

var allSecrets = await FetchAll<SecretListEntry, ListSecretsRequest, ListSecretsResponse> (
				() => new ListSecretsRequest(),
				request => client.ListSecretsAsync(request),
				response => response.NextToken,
				(request, token) => request.NextToken = token,
				response => response.SecretList,
				response => response.NextToken != null
);
								
allSecrets.Dump();

FetchAll.cs

async Task<IReadOnlyList<T>> FetchAll<T, TRequest, TResponse> (
					Func<TRequest> requestFactory,
					Func<TRequest, Task<TResponse>> requestor,
					Func<TResponse, string> tokenGetter,
					Action<TRequest, string> tokenSetter,
					Func<TResponse, IEnumerable<T>> resultExtractor,
					Func<TResponse, bool> hasMore
)
{
	if (requestFactory == null) throw new ArgumentNullException(nameof(requestFactory));
	if (requestor == null) throw new ArgumentNullException(nameof(requestor));
	if (tokenGetter == null) throw new ArgumentNullException(nameof(tokenGetter));
	if (tokenSetter == null) throw new ArgumentNullException(nameof(tokenSetter));
	if (resultExtractor == null) throw new ArgumentNullException(nameof(resultExtractor));
	if (hasMore == null) throw new ArgumentNullException(nameof(hasMore));
	
	TResponse response = default;

	List<T> result = new List<T>();

	do
	{
		var nextToken = response != null ? tokenGetter (response) : null;

		var request = requestFactory();
		tokenSetter (request, nextToken);

		response = await requestor (request);
		
		var items = resultExtractor(response);

		result.AddRange (items);

	} while (hasMore(response));

	return result;
}