Last active
October 12, 2023 05:07
-
-
Save nmvuong92/8bf7dc6962f3034da2c09a03be4ba19c to your computer and use it in GitHub Desktop.
golang s3 download with progressbar
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package awss3 | |
import ( | |
"crypto/tls" | |
"fmt" | |
"io" | |
"net/http" | |
"os" | |
"path/filepath" | |
"time" | |
"github.com/schollz/progressbar/v3" | |
"github.com/aws/aws-sdk-go/aws" | |
"github.com/aws/aws-sdk-go/aws/session" | |
"github.com/aws/aws-sdk-go/service/s3" | |
"github.com/aws/aws-sdk-go/service/s3/s3manager" | |
) | |
type S3Bucket struct { | |
BucketName string | |
PathPrefix string | |
Region string | |
} | |
func (s *S3Bucket) Download(s3filename string, destPath string) (err error) { | |
sess, err := session.NewSession(&aws.Config{ | |
Region: aws.String(s.Region), | |
}) | |
if utils.CheckFileExist(destPath) { | |
_ = os.Remove(destPath) | |
} | |
destFile, err := os.Create(destPath) | |
if err != nil { | |
fmt.Println(err) | |
} | |
defer destFile.Close() | |
s3object, err := s.GetObject(s.PathPrefix + "/" + s3filename) | |
bar := progressbar.DefaultBytes( | |
*s3object.ContentLength, | |
"[info] downloading s3: "+s3filename, | |
) | |
progressbarTicker := time.NewTicker(200 * time.Millisecond) | |
quitProgressBarChannel := make(chan interface{}) | |
go func() { | |
for { | |
select { | |
case <-progressbarTicker.C: | |
localObject, err2 := os.Stat(destPath) | |
if err2 != nil { | |
return | |
} | |
_ = bar.Set64(localObject.Size()) | |
case <-quitProgressBarChannel: | |
progressbarTicker.Stop() | |
return | |
} | |
} | |
}() | |
downloader := s3manager.NewDownloader(sess, func(d *s3manager.Downloader) { | |
d.PartSize = 64 * 1024 * 1024 | |
d.Concurrency = 6 | |
}) | |
_, err = downloader.Download(destFile, | |
&s3.GetObjectInput{ | |
Bucket: aws.String(s.BucketName), | |
Key: aws.String(s.PathPrefix + "/" + s3filename), | |
}) | |
defer func() { | |
time.Sleep(200 * time.Millisecond) | |
quitProgressBarChannel <- true | |
}() | |
if err != nil { | |
fmt.Println(err) | |
} | |
return | |
} | |
func (s *S3Bucket) GetObject(objectKey string) (result *s3.GetObjectOutput, err error) { | |
client := &http.Client{Transport: &http.Transport{ | |
TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, | |
}} | |
sess, err := session.NewSession(&aws.Config{ | |
Region: aws.String(s.Region), | |
}) | |
svc := s3.New(sess, &aws.Config{ | |
HTTPClient: client, | |
}) | |
result, err = svc.GetObject(&s3.GetObjectInput{ | |
Bucket: aws.String(s.BucketName), | |
Key: aws.String(objectKey), | |
}) | |
return | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment