Skip to content

Instantly share code, notes, and snippets.

@coderanger

coderanger/s3.py

Last active August 29, 2015 13:59
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save coderanger/10804335 to your computer and use it in GitHub Desktop.
Save coderanger/10804335 to your computer and use it in GitHub Desktop.
Download ZIP
Unfinished apt-s3 replacement.
Raw
s3.py
#!/usr/bin/python
#
# Author:: Noah Kantrowitz <noah@coderanger.net>
#
# Copyright 2013-2014, Balanced, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import hashlib
import hmac
import json
import sys
import urllib2
import urlparse
class Message(object):
CODES = {
100: 'Capabilities',
102: 'Status',
200: 'URI Start',
201: 'URI Done',
600: 'URI Acquire',
601: 'Configuration',
}
def __init__(self, code, headers, **kwargs):
if code not in self.CODES:
raise ValueError('Unknown code {0}'.format(code))
self.code = code
self.headers = headers + kwargs.items()
@classmethod
def from_lines(cls, lines):
status_line = lines.pop(0)
code = int(status_line.split()[0])
headers = []
for line in lines:
line = line.strip()
if not line:
continue
parts = [p.strip() for p in line.split(':', 1)]
if len(parts) != 2:
continue
headers.append(parts)
return cls(code, headers)
@property
def message(self):
return self.CODES[self.code]
def encode(self):
buf = ['{0} {1}\n'.format(self.code, self.message)]
for key, value in self.headers:
buf.append('{0}: {1}\n'.format(key, value))
buf.append('\n')
return ''.join(buf)
def __getitem__(self, header):
for key, value in self.headers:
if key == header:
return value
class AptS3(object):
MessageClass = Message
def __init__(self, inf=sys.stdin, outf=sys.stdout):
self.inf = inf
self.outf = outf
self.apt_config = {}
self.access_key_id = ''
self.secret_access_key = ''
self.token = None
def read_config(self, path):
try:
data = json.load(open(path, 'wb'))
except (IOError, ValueError):
pass
else:
self.access_key_id = data.get('access_key_id')
self.secret_access_key = data.get('secret_access_key')
self.token = data.get('token')
def send(self, msg, *args, **kwargs):
if not isinstance(msg, self.MessageClass):
msg = self.MessageClass(msg, *args, **kwargs)
self.outf.write(msg.encode())
def send_capabilities(self):
self.send(100, [
('Version', '1.2'),
('Send-Config', 'true'),
])
def read_messages(self):
buf = []
for line in sys.stdin:
if line == '\n':
if buf:
# Decode and handle a message
msg = self.MessageClass.from_lines(buf)
buf = []
self.handle(msg)
else:
buf.append(line)
def handle(self, msg):
fn = getattr(self, 'handle_{0}'.format(msg.code), None)
if not fn:
raise ValueError('No handler for code {0}'.format(msg.code))
return fn(msg)
def handle_600(self, msg):
"""Handle a 600 URI Acquire message."""
def handle_601(self, msg):
"""Handle a 600 Configuration message."""
for key, value in msg.headers:
if key != 'Config-Item':
continue
sub_key, sub_value = value.split('=', 1)
self.apt_config[sub_key] = sub_value
def s3_request(self, url):
pass
if __name__ == '__main__':
apt_s3 = AptS3()
apt_s3.read_config('/etc/apt/s3.json')
apt_s3.send_capabilities()
apt_s3.read_messages()
Raw
z_example_in.txt
601 Configuration
Config-Item: APT::Architecture=amd64
Config-Item: APT::Build-Essential::=build-essential
Config-Item: APT::Install-Recommends=1
Config-Item: APT::Install-Suggests=0
Config-Item: APT::Authentication::TrustCDROM=true
Config-Item: APT::NeverAutoRemove::=^firmware-linux.*
Config-Item: APT::NeverAutoRemove::=^linux-firmware$
Config-Item: APT::NeverAutoRemove::=^kfreebsd-image.*
Config-Item: APT::NeverAutoRemove::=^gnumach$
Config-Item: APT::NeverAutoRemove::=^gnumach-image.*
Config-Item: APT::NeverAutoRemove::=^linux-image-3.5.0-23-generic$
Config-Item: APT::NeverAutoRemove::=^linux-image-extra-3.5.0-23-generic$
Config-Item: APT::NeverAutoRemove::=^linux-signed-image-3.5.0-23-generic$
Config-Item: APT::NeverAutoRemove::=^linux-backports-modules-.*-3.5.0-23-generic$
Config-Item: APT::NeverAutoRemove::=^linux-headers-3.5.0-23-generic$
Config-Item: APT::Never-MarkAuto-Sections::=metapackages
Config-Item: APT::Never-MarkAuto-Sections::=restricted/metapackages
Config-Item: APT::Never-MarkAuto-Sections::=universe/metapackages
Config-Item: APT::Never-MarkAuto-Sections::=multiverse/metapackages
Config-Item: APT::Never-MarkAuto-Sections::=oldlibs
Config-Item: APT::Never-MarkAuto-Sections::=restricted/oldlibs
Config-Item: APT::Never-MarkAuto-Sections::=universe/oldlibs
Config-Item: APT::Never-MarkAuto-Sections::=multiverse/oldlibs
Config-Item: APT::Periodic::Update-Package-Lists=1
Config-Item: APT::Periodic::Unattended-Upgrade=1
Config-Item: APT::Changelogs::Server=http://changelogs.ubuntu.com/changelogs
Config-Item: Dir=/
Config-Item: Dir::State=var/lib/apt/
Config-Item: Dir::State::lists=lists/
Config-Item: Dir::State::cdroms=cdroms.list
Config-Item: Dir::State::mirrors=mirrors/
Config-Item: Dir::State::extended_states=extended_states
Config-Item: Dir::State::status=/var/lib/dpkg/status
Config-Item: Dir::Cache=var/cache/apt/
Config-Item: Dir::Cache::archives=archives/
Config-Item: Dir::Cache::srcpkgcache=srcpkgcache.bin
Config-Item: Dir::Cache::pkgcache=pkgcache.bin
Config-Item: Dir::Etc=etc/apt/
Config-Item: Dir::Etc::sourcelist=sources.list
Config-Item: Dir::Etc::sourceparts=sources.list.d
Config-Item: Dir::Etc::vendorlist=vendors.list
Config-Item: Dir::Etc::vendorparts=vendors.list.d
Config-Item: Dir::Etc::main=apt.conf
Config-Item: Dir::Etc::netrc=auth.conf
Config-Item: Dir::Etc::parts=apt.conf.d
Config-Item: Dir::Etc::preferences=preferences
Config-Item: Dir::Etc::preferencesparts=preferences.d
Config-Item: Dir::Etc::trusted=trusted.gpg
Config-Item: Dir::Etc::trustedparts=trusted.gpg.d
Config-Item: Dir::Bin::methods=/usr/lib/apt/methods
Config-Item: Dir::Bin::solvers::=/usr/lib/apt/solvers
Config-Item: Dir::Bin::dpkg=/usr/bin/dpkg
Config-Item: Dir::Media::MountPath=/media/cdrom
Config-Item: Dir::Log=var/log/apt
Config-Item: Dir::Log::Terminal=term.log
Config-Item: Dir::Log::History=history.log
Config-Item: Dir::Ignore-Files-Silently::=~$
Config-Item: Dir::Ignore-Files-Silently::=\.disabled$
Config-Item: Dir::Ignore-Files-Silently::=\.bak$
Config-Item: Dir::Ignore-Files-Silently::=\.dpkg-[a-z]+$
Config-Item: Dir::Ignore-Files-Silently::=\.save$
Config-Item: Dir::Ignore-Files-Silently::=\.orig$
Config-Item: Dir::Ignore-Files-Silently::=\.distUpgrade$
Config-Item: Acquire::cdrom::mount=/media/cdrom
Config-Item: Aptitude::Get-Root-Command=sudo:/usr/bin/sudo
Config-Item: Aptitude::Keep-Unused-Pattern=^linux-image.*$%20|%20^linux-restricted-modules.*$%20|%20^linux-ubuntu-modules.*$
Config-Item: Unattended-Upgrade::Allowed-Origins::=${distro_id}:${distro_codename}-security
Config-Item: DPkg::Pre-Install-Pkgs::=/usr/sbin/dpkg-preconfigure%20--apt%20||%20true
Config-Item: CommandLine::AsString=apt-get%20update
600 URI Acquire
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/Release.gpg
Filename: /var/lib/apt/lists/partial/extras.ubuntu.com_ubuntu_dists_precise_Release.gpg
Index-File: true
600 URI Acquire
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/Release
Filename: /var/lib/apt/lists/partial/extras.ubuntu.com_ubuntu_dists_precise_Release
Index-File: true
600 URI Acquire
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/binary-amd64/Packages.bz2
Filename: /var/lib/apt/lists/partial/extras.ubuntu.com_ubuntu_dists_precise_main_binary-amd64_Packages
Index-File: true
Fail-Ignore: true
600 URI Acquire
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/binary-i386/Packages.bz2
Filename: /var/lib/apt/lists/partial/extras.ubuntu.com_ubuntu_dists_precise_main_binary-i386_Packages
Index-File: true
Fail-Ignore: true
600 URI Acquire
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Index
Filename: /var/lib/apt/lists/partial/extras.ubuntu.com_ubuntu_dists_precise_main_i18n_Index
Index-File: true
Fail-Ignore: true
600 URI Acquire
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en_US.bz2
Filename: /var/lib/apt/lists/partial/extras.ubuntu.com_ubuntu_dists_precise_main_i18n_Translation-en%5fUS
Index-File: true
Fail-Ignore: true
600 URI Acquire
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en.bz2
Filename: /var/lib/apt/lists/partial/extras.ubuntu.com_ubuntu_dists_precise_main_i18n_Translation-en
Fail-Ignore: true
Index-File: true
600 URI Acquire
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en_US.xz
Filename: /var/lib/apt/lists/partial/extras.ubuntu.com_ubuntu_dists_precise_main_i18n_Translation-en%5fUS
Fail-Ignore: true
Index-File: true
600 URI Acquire
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en.xz
Filename: /var/lib/apt/lists/partial/extras.ubuntu.com_ubuntu_dists_precise_main_i18n_Translation-en
Fail-Ignore: true
Index-File: true
600 URI Acquire
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en_US.lzma
Filename: /var/lib/apt/lists/partial/extras.ubuntu.com_ubuntu_dists_precise_main_i18n_Translation-en%5fUS
Fail-Ignore: true
Index-File: true
600 URI Acquire
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en.lzma
Filename: /var/lib/apt/lists/partial/extras.ubuntu.com_ubuntu_dists_precise_main_i18n_Translation-en
Fail-Ignore: true
Index-File: true
600 URI Acquire
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en_US.gz
Filename: /var/lib/apt/lists/partial/extras.ubuntu.com_ubuntu_dists_precise_main_i18n_Translation-en%5fUS
Fail-Ignore: true
Index-File: true
600 URI Acquire
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en.gz
Filename: /var/lib/apt/lists/partial/extras.ubuntu.com_ubuntu_dists_precise_main_i18n_Translation-en
Fail-Ignore: true
Index-File: true
600 URI Acquire
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en_US
Filename: /var/lib/apt/lists/partial/extras.ubuntu.com_ubuntu_dists_precise_main_i18n_Translation-en%5fUS
Fail-Ignore: true
Index-File: true
600 URI Acquire
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en
Filename: /var/lib/apt/lists/partial/extras.ubuntu.com_ubuntu_dists_precise_main_i18n_Translation-en
Fail-Ignore: true
Index-File: true
Raw
z_example_out.txt
100 Capabilities
Version: 1.2
Pipeline: true
Send-Config: true
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/Release.gpg
Message: Connecting to extras.ubuntu.com
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/Release.gpg
Message: Connecting to extras.ubuntu.com (91.189.92.152)
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/Release.gpg
Message: Waiting for headers
root@balanced-apt-berkshelf:/usr/lib/apt/methods# (cat /tmp/foo && cat) | ./http
100 Capabilities
Version: 1.2
Pipeline: true
Send-Config: true
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/Release.gpg
Message: Connecting to extras.ubuntu.com
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/Release.gpg
Message: Connecting to extras.ubuntu.com (91.189.92.152)
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/Release.gpg
Message: Waiting for headers
200 URI Start
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/Release.gpg
Size: 72
Last-Modified: Mon, 20 Jan 2014 16:01:02 GMT
Resume-Point: 71
201 URI Done
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/Release.gpg
Filename: /var/lib/apt/lists/partial/extras.ubuntu.com_ubuntu_dists_precise_Release.gpg
Size: 72
Last-Modified: Mon, 20 Jan 2014 16:01:02 GMT
MD5-Hash: 947d0e7cba203ac7cf1255995428dfd2
MD5Sum-Hash: 947d0e7cba203ac7cf1255995428dfd2
SHA1-Hash: c29199d25b5cad0691bc75fe9477130d38c96ea5
SHA256-Hash: c74d3b367cc9a6eb68823555e25d0f97530c07a7f7edcf2e1dbe35a74f4263ea
SHA512-Hash: 2df39c219508c6cc274c993fafe6515a3f605cfe85013a91b80dfd16569c3f1cb2586e004df794267ee9c7c68f038009e053940c40e4d5669f09ce7347a2e7d3
Resume-Point: 71
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/Release
Message: Waiting for headers
200 URI Start
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/Release
Size: 11875
Last-Modified: Fri, 15 Feb 2013 15:05:55 GMT
201 URI Done
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/Release
Filename: /var/lib/apt/lists/partial/extras.ubuntu.com_ubuntu_dists_precise_Release
Size: 11875
Last-Modified: Fri, 15 Feb 2013 15:05:55 GMT
MD5-Hash: 5518a6c60c415380f37fc64ecb069904
MD5Sum-Hash: 5518a6c60c415380f37fc64ecb069904
SHA1-Hash: 5771f2c92b2e5c38487722950a1cdcfc8f82b405
SHA256-Hash: a3ea54a7f398ce1b4f8f271f34349cc89c9423e3acab7fcabf8204acf1d17439
SHA512-Hash: dda4564733e9463dc48b7e0cf4a37ecb414b3a10dd201b63f303b68e8c94adc5b5845f01a2764dcd8898010615be43b9c374bedcac279341ec583f0f98694c15
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/binary-amd64/Packages.bz2
Message: Waiting for headers
200 URI Start
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/binary-amd64/Packages.bz2
Size: 10788
Last-Modified: Fri, 15 Feb 2013 16:01:01 GMT
201 URI Done
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/binary-amd64/Packages.bz2
Filename: /var/lib/apt/lists/partial/extras.ubuntu.com_ubuntu_dists_precise_main_binary-amd64_Packages
Size: 10788
Last-Modified: Fri, 15 Feb 2013 16:01:01 GMT
MD5-Hash: 4019d042f1f2fe7644056a366b3364f1
MD5Sum-Hash: 4019d042f1f2fe7644056a366b3364f1
SHA1-Hash: 807d780e437efcd456ec9302504b04f1edfdb13f
SHA256-Hash: 41bb5837826fd66d37b5c520a948f5f50a9abf1e317d1d53f46d506260b85db7
SHA512-Hash: 92438637ce9e1c072c48a7ad2793c9255a6126ce5fb7f719f7d21db14b52b8d3a74c8656de184c64d7fa362893dddf30784f642e0b2d78f67ab81a3cda1b6f8a
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/binary-i386/Packages.bz2
Message: Waiting for headers
200 URI Start
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/binary-i386/Packages.bz2
Size: 10788
Last-Modified: Fri, 15 Feb 2013 16:01:01 GMT
201 URI Done
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/binary-i386/Packages.bz2
Filename: /var/lib/apt/lists/partial/extras.ubuntu.com_ubuntu_dists_precise_main_binary-i386_Packages
Size: 10788
Last-Modified: Fri, 15 Feb 2013 16:01:01 GMT
MD5-Hash: a7111e0bc5e918b8ef316ef933bf6e23
MD5Sum-Hash: a7111e0bc5e918b8ef316ef933bf6e23
SHA1-Hash: 3556d8d91cf3c102dd7e338e447df4c5d820ecb7
SHA256-Hash: 203da9b31b1bae2e808bd68b5fdee7ee2c09c6d18ea00e191c683459f3fa772e
SHA512-Hash: 561471fab19d21b15f3f1d3ce87e854604cb50d2f367eb8d0b7373787ad0fda32d9fe7f44a750376b9cac3b6113df400afbd47b44a88b66170c12250d8e1c989
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Index
Message: Waiting for headers
400 URI Failure
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Index
Message: 404 Not Found
FailReason: HttpError404
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en_US.bz2
Message: Waiting for headers
400 URI Failure
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en_US.bz2
Message: 404 Not Found
FailReason: HttpError404
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en.bz2
Message: Waiting for headers
400 URI Failure
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en.bz2
Message: 404 Not Found
FailReason: HttpError404
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en_US.xz
Message: Waiting for headers
400 URI Failure
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en_US.xz
Message: 404 Not Found
FailReason: HttpError404
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en.xz
Message: Waiting for headers
400 URI Failure
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en.xz
Message: 404 Not Found
FailReason: HttpError404
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en_US.lzma
Message: Waiting for headers
400 URI Failure
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en_US.lzma
Message: 404 Not Found
FailReason: HttpError404
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en.lzma
Message: Waiting for headers
400 URI Failure
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en.lzma
Message: 404 Not Found
FailReason: HttpError404
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en_US.gz
Message: Waiting for headers
400 URI Failure
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en_US.gz
Message: 404 Not Found
FailReason: HttpError404
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en.gz
Message: Waiting for headers
400 URI Failure
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en.gz
Message: 404 Not Found
FailReason: HttpError404
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en_US
Message: Waiting for headers
400 URI Failure
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en_US
Message: 404 Not Found
FailReason: HttpError404
102 Status
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en
Message: Waiting for headers
400 URI Failure
URI: foo://extras.ubuntu.com/ubuntu/dists/precise/main/i18n/Translation-en
Message: 404 Not Found
FailReason: HttpError404
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment