Skip to content

Instantly share code, notes, and snippets.

@nmvuong92
Last active October 12, 2023 05:07
Show Gist options
  • Star 3 You must be signed in to star a gist
  • Fork 2 You must be signed in to fork a gist
  • Save nmvuong92/8bf7dc6962f3034da2c09a03be4ba19c to your computer and use it in GitHub Desktop.
Save nmvuong92/8bf7dc6962f3034da2c09a03be4ba19c to your computer and use it in GitHub Desktop.
golang s3 download with progressbar
package awss3
import (
"crypto/tls"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"time"
"github.com/schollz/progressbar/v3"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
)
type S3Bucket struct {
BucketName string
PathPrefix string
Region string
}
func (s *S3Bucket) Download(s3filename string, destPath string) (err error) {
sess, err := session.NewSession(&aws.Config{
Region: aws.String(s.Region),
})
if utils.CheckFileExist(destPath) {
_ = os.Remove(destPath)
}
destFile, err := os.Create(destPath)
if err != nil {
fmt.Println(err)
}
defer destFile.Close()
s3object, err := s.GetObject(s.PathPrefix + "/" + s3filename)
bar := progressbar.DefaultBytes(
*s3object.ContentLength,
"[info] downloading s3: "+s3filename,
)
progressbarTicker := time.NewTicker(200 * time.Millisecond)
quitProgressBarChannel := make(chan interface{})
go func() {
for {
select {
case <-progressbarTicker.C:
localObject, err2 := os.Stat(destPath)
if err2 != nil {
return
}
_ = bar.Set64(localObject.Size())
case <-quitProgressBarChannel:
progressbarTicker.Stop()
return
}
}
}()
downloader := s3manager.NewDownloader(sess, func(d *s3manager.Downloader) {
d.PartSize = 64 * 1024 * 1024
d.Concurrency = 6
})
_, err = downloader.Download(destFile,
&s3.GetObjectInput{
Bucket: aws.String(s.BucketName),
Key: aws.String(s.PathPrefix + "/" + s3filename),
})
defer func() {
time.Sleep(200 * time.Millisecond)
quitProgressBarChannel <- true
}()
if err != nil {
fmt.Println(err)
}
return
}
func (s *S3Bucket) GetObject(objectKey string) (result *s3.GetObjectOutput, err error) {
client := &http.Client{Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}}
sess, err := session.NewSession(&aws.Config{
Region: aws.String(s.Region),
})
svc := s3.New(sess, &aws.Config{
HTTPClient: client,
})
result, err = svc.GetObject(&s3.GetObjectInput{
Bucket: aws.String(s.BucketName),
Key: aws.String(objectKey),
})
return
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment