Skip to content

Instantly share code, notes, and snippets.

View wthorp's full-sized avatar

Bill Thorp wthorp

13 followers · 2 following
  • storj.io
View GitHub Profile
@wthorp
wthorp / sizes.go
Created September 7, 2023 20:01
comparison of calculating prefix sizes with S3 and Storj native using GC functions
package cloudsigning
import (
"fmt"
"net/http"
"github.com/GoogleCloudPlatform/functions-framework-go/functions"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
@wthorp
wthorp / create_client_creds.py
Created August 14, 2023 16:08
import requests
from uplink_python.uplink import Uplink
from uplink_python.module_classes import Permission, SharePrefix
API_KEY = "..."
ENC_PASSWORD = "..."
SATELLITE_HOST = "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S@us1.storj.io:7777"
CUSTOMER_NAME = "clientA"
BUCKET = "client-data"
@wthorp
wthorp / presigned_post.py
Created August 14, 2023 16:07
import boto3
ACCESS_KEY = "..."
SECRET_KEY = "..."
BUCKET_NAME = "..."
PATH_WITHIN_BUCKET = "example/path"
EXPIRES = 3600 # seconds
URL = "https://gateway.storjshare.io"
session = boto3.session.Session()
@wthorp
wthorp / presigned_get.py
Created August 14, 2023 16:06
Presigned GET Python
import boto3
ACCESS_KEY = "..."
SECRET_KEY = "..."
BUCKET_NAME = "..."
PATH_WITHIN_BUCKET = "example/path"
EXPIRES = 3600 # seconds
URL = "https://gateway.storjshare.io"
session = boto3.session.Session()
@wthorp
wthorp / presigned.go
Created February 24, 2023 17:49
WIP AWS Lambda Presigned URLs
package main
import (
"fmt"
"time"
"github.com/aws/aws-lambda-go/lambda"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/request"
@wthorp
wthorp / cloudsigning.go
Created February 14, 2023 17:00
Google Cloud Function Presigned URLs
package cloudsigning
import (
"fmt"
"net/http"
"time"
"github.com/GoogleCloudPlatform/functions-framework-go/functions"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
@wthorp
wthorp / lsdl2.go
Created July 14, 2022 23:06
parallel downloader take 2
// Copyright (C) 2022 Storj Labs, Inc.
// See LICENSE for copying information.
package main
import (
"context"
"flag"
"fmt"
"io/ioutil"
@wthorp
wthorp / main.go
Created July 6, 2022 03:53
parallel downloader suitable for piping
// Copyright (C) 2022 Storj Labs, Inc.
// See LICENSE for copying information.
package main
import (
"flag"
"fmt"
"io/ioutil"
"net/http"
@wthorp
wthorp / copy.py
Created May 27, 2022 12:16
Force S3 CopyObject over PutPartCopy
import boto3
import os
from boto3.s3.transfer import TransferConfig
# Set the desired multipart threshold to an arbitrarily large value.
EB = 1024 ** 6
config = TransferConfig(multipart_threshold=EB)
session = boto3.session.Session()
s3_client = session.client(
@wthorp
wthorp / gist:8dab156cfbb0f451eeb0954a35fd1c46
Last active April 27, 2022 11:06
Using Storj from AWS SDK for Go without the HTTP
// NoIOTransport can be used as an http.Client{Transport: NewNoIOTransport()},
// EG: s3.New(awsSession, aws.NewConfig().WithHTTPClient(&http.Client{Transport: NewNoIOTransport()}))
// downloader := s3manager.NewDownloader(awsSession, func(d *s3manager.Downloader) {
// d.S3 = s3.New(awsSession, aws.NewConfig().WithHTTPClient(&http.Client{Transport: NewNoIOTransport()}))
// })
// uploader := s3manager.NewUploader(awsSession, func(u *s3manager.Uploader) {
// u.S3 = s3.New(awsSession, aws.NewConfig().WithHTTPClient(&http.Client{Transport: NewNoIOTransport()}))
// })
type NoIOTransport struct {
r *mux.Router