Skip to content

Instantly share code, notes, and snippets.

@mimoo
Last active March 5, 2024 03:53
Show Gist options
  • Star 29 You must be signed in to star a gist
  • Fork 10 You must be signed in to fork a gist
  • Save mimoo/25fc9716e0f1353791f5908f94d6e726 to your computer and use it in GitHub Desktop.
Save mimoo/25fc9716e0f1353791f5908f94d6e726 to your computer and use it in GitHub Desktop.
How to compress a folder in Golang using tar and gzip (works with nested folders)
package main
import (
"archive/tar"
"bytes"
"compress/gzip"
"fmt"
"io"
"os"
"path/filepath"
)
func main() {
// tar + gzip
var buf bytes.Buffer
_ := compress("./folderToCompress", &buf)
// write the .tar.gzip
fileToWrite, err := os.OpenFile("./compress.tar.gzip", os.O_CREATE|os.O_RDWR, os.FileMode(600))
if err != nil {
panic(err)
}
if _, err := io.Copy(fileToWrite, &buf); err != nil {
panic(err)
}
// untar write
if err := untar(&buf, "./uncompressHere/"); err != nil {
// probably delete uncompressHere?
}
}
func compress(src string, buf io.Writer) error {
// tar > gzip > buf
zr := gzip.NewWriter(buf)
tw := tar.NewWriter(zr)
// walk through every file in the folder
filepath.Walk(src, func(file string, fi os.FileInfo, err error) error {
// generate tar header
header, err := tar.FileInfoHeader(fi, file)
if err != nil {
return err
}
// must provide real name
// (see https://golang.org/src/archive/tar/common.go?#L626)
header.Name = filepath.ToSlash(file)
// write header
if err := tw.WriteHeader(header); err != nil {
return err
}
// if not a dir, write file content
if !fi.IsDir() {
data, err := os.Open(file)
if err != nil {
return err
}
if _, err := io.Copy(tw, data); err != nil {
return err
}
}
return nil
})
// produce tar
if err := tw.Close(); err != nil {
return err
}
// produce gzip
if err := zr.Close(); err != nil {
return err
}
//
return nil
}
// check for path traversal and correct forward slashes
func validRelPath(p string) bool {
if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") {
return false
}
return true
}
func uncompress(src io.Reader, dst string) error {
// ungzip
zr, err := gzip.NewReader(src)
if err != nil {
return err
}
// untar
tr := tar.NewReader(zr)
// uncompress each element
for {
header, err := tr.Next()
if err == io.EOF {
break // End of archive
}
if err != nil {
return err
}
target :=
// validate name against path traversal
if !validRelPath(header.Name) {
return fmt.Errorf("tar contained invalid name error %q\n", target)
}
// add dst + re-format slashes according to system
target = filepath.Join(dst, header.Name)
// if no join is needed, replace with ToSlash:
// target = filepath.ToSlash(header.Name)
// check the type
switch header.Typeflag {
// if its a dir and it doesn't exist create it (with 0755 permission)
case tar.TypeDir:
if _, err := os.Stat(target); err != nil {
if err := os.MkdirAll(target, 0755); err != nil {
return err
}
}
// if it's a file create it (with same permission)
case tar.TypeReg:
fileToWrite, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
if err != nil {
return err
}
// copy over contents
if _, err := io.Copy(fileToWrite, tr); err != nil {
return err
}
// manually close here after each file operation; defering would cause each file close
// to wait until all operations have completed.
fileToWrite.Close()
}
}
}
@Templum
Copy link

Templum commented Oct 20, 2020

Missing target initialization on line L106

@mimoo
Copy link
Author

mimoo commented Oct 20, 2020

sorry -.- you can see how I did it for real in eureka

@Templum
Copy link

Templum commented Oct 20, 2020

@mimoo thanks for sharing :)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment