A favorite interview question of mine... how to run an async batch job against a large list of objects

csharp-batching-interview-question.cs

public class Person {
  public string FirstName { get; set; }
  public string LastName { get;set; }
  public int Age { get; set; }

  public async Task Finalize() {
    // does something does not matter what
  }
}

public class PersonService {
  public async Task<List<Person>> GetPeople() {
    // creates the initial connection to operate against the database
    using (var connection = new FancyYetNonOrmLibrary(Configuration.Constants)) {
      // returns a data table contained the results of the query
      DataTable data = await GetDataTable("select first_name, last_name, age from people")
      
      // so we have out list from the first questions, the result contains 250k or 500k records (ie a lot)
      // List<Person> people = ...our last questions result
      List<Person> people = data.AsEnumerable().Select(x => new Person {
        //
        FirstName = x["first_name"].ToString(),
        LastName = x["last_name"].ToString(),
        Age = Convert.Int32(x["age"])
      }).ToList();

      // TODO...using async/await perform a batch operation to execute the Finalize() method on each object
      // ... using no more than 10 threads or tasks for the operation
      // ... then return the final result
      
      // the answer
 
      // var for tasks
      List<Task> tasks = new List<Task>();
      // final output list
      List<Person> finalizedPeople = new List<Person>();
      
      // chunk the list
      List<List<Person>> chunkedPeople = people
                .Select((x, i) => new { Index = i, Value = x })
                .GroupBy(x => x.Index / 10)
                .Select(x => x.Select(v => v.Value).ToList())
                .ToList();
      
      // iterate through the chunks
      foreach (List<Person> peopleChunk in chunkedPeople)
      {
        // add a task to the task list for each chunk
        task.Add(Task.Run async () => {
          // loop through all the people in the chunk
          foreach (List<int> item in chunkList)
          {
              // execute the method
              finalizedPeople.Add(await item.Finalize());
          }
        });
      }
      
      // wait for all the tasks in the task list to finish
      await Task.WhenAll(tasks);
      
      // return the result
      return finalizedPeople;
    }
  }
}