Part of https://jacoposalvestrini.medium.com/batch-download-files-from-aws-s3-36e575f30682

AWSS3-BatchDownload-main.go

const localBase = "/tmp/"

func download(ctx context.Context, client *s3.S3, bucket string, paths []string) ([]string, error) {

	downloadedFiles := []string{}
	length := len(paths)
	if length < 1 {
		return downloadedFiles, fmt.Errorf("no files to download")
	}

	forDownload := []s3manager.BatchDownloadObject{}
	for _, destFilename := range paths {

		// Skip directories
		if strings.HasSuffix(destFilename, "/") {
			continue
		}

		// Mirroring S3 directory structure locally
		if strings.Contains(destFilename, "/") {
			var dirTree string
			// split
			s3FileFullPathList := strings.Split(destFilename, "/")
			for _, dir := range s3FileFullPathList[:len(s3FileFullPathList)-1] {
				dirTree += "/" + dir
			}
			os.MkdirAll(localBase+dirTree, 0775)
		}

		destFilePath := localBase + destFilename

		// Create batch download objects and add them to the forDownload list
		forDownload = append(forDownload, s3manager.BatchDownloadObject{
			Object: &s3.GetObjectInput{
				Bucket: aws.String(bucket),
				Key:    aws.String(destFilename),
			},
			// Prepare for saving to an in-memory cache
			Writer: &buffer{Path: destFilePath},
		})
	}

	// Eventually saving to files
	defer func() {
		for _, downloadObject := range forDownload {
			buf, ok := downloadObject.Writer.(*buffer)
			if !ok {
				// log something
				continue
			}
			n, err := buf.Save()
			if err != nil {
				// log something
				continue
			}
			if n < 1 {
				// log something
				continue
			}
			downloadedFiles = append(downloadedFiles, buf.Path)
		}
		// log that len(downloadedFiles) files have been downloaded
	}()

	// Run the iterator
	err := s3manager.NewDownloaderWithClient(client)
		.DownloadWithIterator(ctx, &s3manager.DownloadObjectsIterator{Objects: forDownload})

	return downloadedFiles, err
}