Skip to content

Instantly share code, notes, and snippets.

@smoser
Last active August 31, 2022 15:44
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save smoser/dc0996f49c0902340e08033465049d0e to your computer and use it in GitHub Desktop.
Save smoser/dc0996f49c0902340e08033465049d0e to your computer and use it in GitHub Desktop.
A deb upload client for artifactory

deb-artifactory - publish a deb to artifactory

This is an example client to upload debs to artifactory.

Example usage is pretty simple:

$ export ARTIFACTORY_CREDS=AKCp.........BuV
$ ./deb-artifactory https://smoser0.jfrog.io/artifactory/test1 my.deb

Things this does:

  • publishes to pool/[src-package]/[version]/[name.deb] where name is set correctly based on metadata in the deb.
  • checks for existing before publishing. Packages of a given version should never change.
  • publish multiple debs

Problems

  • Cannot copy between distributions after publication. [jfrog RTFACT-27075]

    'Distribution' is the term that artifactory uses for a 'suite' (sources.list(5)). Common examples of suite are 'focal', 'focal-updates', 'focal-security', 'focal-proposed'.

    When you upload you can specify multiple distributions with "matrix parameters". But you cannot subsequently add distributions. The workflow that is blocked by this missing feature is the ability to "copy" from a "staging" distribution (like 'focal-proposed') to a production distribution (like 'focal-updates').

module deb-artifactory
go 1.18
require pault.ag/go/debian v0.12.0
require (
github.com/DataDog/zstd v1.4.8 // indirect
github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect
pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a // indirect
)
github.com/DataDog/zstd v1.4.8 h1:Rpmta4xZ/MgZnriKNd24iZMhGpP5dvUcs/uqfBapKZY=
github.com/DataDog/zstd v1.4.8/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d h1:RnWZeH8N8KXfbwMTex/KKMYMj0FJRCF6tQubUuQ02GM=
github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d/go.mod h1:phT/jsRPBAEqjAibu1BurrabCBNTYiVI+zbmyCZJY6Q=
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f h1:OeJjE6G4dgCY4PIXvIRQbE8+RX+uXZyGhUy/ksMGJoc=
golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
pault.ag/go/debian v0.12.0 h1:b8ctSdBSGJ98NE1VLn06aSx70EUpczlP2qqSHEiYYJA=
pault.ag/go/debian v0.12.0/go.mod h1:UbnMr3z/KZepjq7VzbYgBEfz8j4+Pyrm2L5X1fzhy/k=
pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a h1:WwS7vlB5H2AtwKj1jsGwp2ZLud1x6WXRXh2fXsRqrcA=
pault.ag/go/topsort v0.0.0-20160530003732-f98d2ad46e1a/go.mod h1:INqx0ClF7kmPAMk2zVTX8DRnhZ/yaA/Mg52g8KFKE7k=
package main
import (
"crypto/sha256"
"flag"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"strings"
"pault.ag/go/debian/deb"
)
func getName(ctrl deb.Control) (string, error) {
if ctrl.Package == "" {
return "", fmt.Errorf("Empty Package")
}
if ctrl.Architecture.CPU == "" {
return "", fmt.Errorf("Missing arch")
}
return fmt.Sprintf("%s_%s_%s.deb", ctrl.Package, ctrl.Version, ctrl.Architecture.CPU), nil
}
func getSha256(fpath string) (string, error) {
f, err := os.Open(fpath)
if err != nil {
return "", err
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return "", err
}
return fmt.Sprintf("%x", h.Sum(nil)), nil
}
func putDeb(repoURL, fpath, suite, component, creds string) error {
client := http.Client{}
if creds == "" {
return fmt.Errorf("empty creds. sorry")
}
debInfo, _, err := deb.LoadFile(fpath)
// defer closer()
if err != nil {
return err
}
ctrl := debInfo.Control
src := ctrl.Source
// Source is only present in metadata if it differs from Package.
if src == "" {
if ctrl.Package == "" {
return fmt.Errorf("empty 'Source' and no 'Package' in %s", fpath)
}
src = ctrl.Package
}
ver := ctrl.Version.Version
if ver == "" {
return fmt.Errorf("bad (empty) version in %s", fpath)
}
/*
stza, err := deb.GetControlFileFromDeb(fpath)
if err != nil {
return err
}
*/
fp, err := os.Open(fpath)
if err != nil {
return err
}
defer fp.Close()
fInfo, err := fp.Stat()
if err != nil {
return err
}
localSize := fInfo.Size()
sha256, err := getSha256(fpath)
if err != nil {
return err
}
name, err := getName(ctrl)
if err != nil {
return err
}
// name = path.Base(fpath)
arch := ctrl.Architecture.CPU
// https://www.jfrog.com/confluence/display/JFROG/Debian+Repositories
pubUrl := strings.TrimRight(repoURL, "/") + "/pool/" + src + "/" + ctrl.Version.String() + "/" + name
matrixParms := strings.Join(
[]string{
"deb.distribution=" + suite,
"deb.component=" + component,
"deb.architecture=" + arch,
}, ";")
req, err := http.NewRequest(http.MethodHead, pubUrl, nil)
toks := strings.SplitN(creds, ":", 1)
if toks[0] == "" {
// empty creds... assume it is in https://user@pass:...
} else if len(toks) == 1 {
req.Header.Set("X-JFrog-Art-Api", creds)
} else {
req.SetBasicAuth(toks[0], toks[1])
}
resp, err := client.Do(req)
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("failed readAll HEAD: %v", err)
}
if resp.StatusCode == http.StatusOK {
fmt.Printf("HEAD [%s] %s\n", resp.Status, pubUrl)
rSha256 := resp.Header["X-Checksum-Sha256"][0]
fmt.Printf("remote:\n size: %d\n sha256: %s\n", resp.ContentLength, rSha256)
fmt.Printf("local:\n size: %d\n sha256: %s\n", localSize, sha256)
if localSize != resp.ContentLength {
return fmt.Errorf("local size %d != remote %d for %s\n", localSize, resp.ContentLength, pubUrl)
}
if sha256 != rSha256 {
return fmt.Errorf("local sha256 %s != remote %s for %s\n", sha256, rSha256, pubUrl)
}
// for publishing to new suite
fmt.Printf("Existing matched local, will upload again.\n")
} else if resp.StatusCode == http.StatusNotFound {
fmt.Printf("New upload. Did not exist\n")
} else {
return fmt.Errorf("Got unexpectd status code %d on HEAD", resp.StatusCode)
}
req, err = http.NewRequest(http.MethodPut, pubUrl+";"+matrixParms, fp)
if err != nil {
return err
}
toks = strings.SplitN(creds, ":", 1)
if toks[0] == "" {
// empty creds... assume it is in https://user@pass:...
} else if len(toks) == 1 {
req.Header.Set("X-JFrog-Art-Api", creds)
} else {
req.SetBasicAuth(toks[0], toks[1])
}
req.Header.Set("X-Checksum-Sha256", sha256)
resp, err = client.Do(req)
if err != nil {
return err
}
fmt.Printf("Posted [%s] %s -> %s\n", resp.Status, fpath, pubUrl)
b, err = ioutil.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("failed readAll: %v", err)
}
fmt.Printf("%s\n", string(b))
return nil
}
func showDeb(fpath string) error {
debInfo, closer, err := deb.LoadFile(fpath)
defer closer()
if err != nil {
return err
}
ver := debInfo.Control.Version
fmt.Printf("Version:\n epoch: %d\n version: %s\n revision: %s\n str: %s\n",
ver.Epoch, ver.Version, ver.Revision, ver.String())
return nil
}
func main() {
const credEnvName = "ARTIFACTORY_CREDS"
const component = "main"
var creds, suite string
flag.StringVar(&creds, "creds", "", "api token or user:pass for artifactory (env "+credEnvName+"_")
flag.StringVar(&suite, "suite", "focal", "suite to publish to (focal)")
flag.Bool("v", false, "verbose")
flag.Parse()
args := flag.Args()
if len(args) <= 1 {
fmt.Printf("Must give args [repo, debs]\n")
os.Exit(1)
}
repo := args[0]
if !strings.HasSuffix(repo, "/") {
repo += "/"
}
if creds == "" {
creds = os.Getenv(credEnvName)
if creds == "" {
fmt.Printf("Must provide creds either --creds= or env %s\n", credEnvName)
}
}
for _, p := range args[1:] {
// showDeb(p)
if err := putDeb(repo, p, suite, component, creds); err != nil {
fmt.Printf("Failed putting deb %s: %v\n", p, err)
os.Exit(1)
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment