Skip to content

Instantly share code, notes, and snippets.

@psachin

psachin/0001-Proxy-server-controller-tests-should-be-reorganized.patch

Created February 23, 2017 08:45
Show Gist options
  • Save psachin/5b8e049aa635ff96e7af362c15c289f6 to your computer and use it in GitHub Desktop.
Save psachin/5b8e049aa635ff96e7af362c15c289f6 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
0001-Proxy-server-controller-tests-should-be-reorganized.patch
From 16c36ce82259fdf82686492d3447dc84b2070ee7 Mon Sep 17 00:00:00 2001
From: Sachin Patil <psachin@redhat.com>
Date: Wed, 28 Dec 2016 22:28:19 +0000
Subject: [PATCH] Proxy server controller tests should be reorganized
Following tests were moved:
- @patch_policies within test_server.TestObjectController -> controllers/test_obj.py
- @patch_policies within test_server.TestContainerController -> controllers/test_container.py
- @patch_policies within test_server.TestAccountController -> controllers/test_account.py
- Lots of refactor and cleanup
Closes-Bug: #1646963
Change-Id: Ib7affafcf7c13193b523e983711461f153a6bc93
Signed-off-by: Sachin Patil <psachin@redhat.com>
---
test/unit/common/middleware/test_copy.py | 16 +-
test/unit/proxy/__init__.py | 71 +
test/unit/proxy/controllers/test_account.py | 714 ++++++-
test/unit/proxy/controllers/test_container.py | 1314 ++++++++++++-
test/unit/proxy/controllers/test_obj.py | 2477 +++++++++++++++++++++++--
test/unit/proxy/test_mem_server.py | 8 +-
test/unit/proxy/test_server.py | 2186 +---------------------
7 files changed, 4436 insertions(+), 2350 deletions(-)
diff --git a/test/unit/common/middleware/test_copy.py b/test/unit/common/middleware/test_copy.py
index 2dba7194ad73..695bbe8d769d 100644
--- a/test/unit/common/middleware/test_copy.py
+++ b/test/unit/common/middleware/test_copy.py
@@ -29,8 +29,8 @@ from swift.common.storage_policy import POLICIES
from swift.common.swob import Request, HTTPException
from test.unit import patch_policies, debug_logger, FakeMemcache, FakeRing
from test.unit.common.middleware.helpers import FakeSwift
-from test.unit.proxy.controllers.test_obj import set_http_connect, \
- PatchedObjControllerApp
+from test.unit.proxy.controllers.test_obj import (
+ set_http_connect_contextmgr, PatchedObjControllerApp)
class TestCopyConstraints(unittest.TestCase):
@@ -1570,9 +1570,10 @@ class TestServerSideCopyMiddlewareWithEC(unittest.TestCase):
if method == 'PUT':
put_hdrs.append(args[0])
- with set_http_connect(*status_codes, body_iter=body_iter,
- headers=headers, expect_headers=expect_headers,
- give_connect=capture_conn):
+ with set_http_connect_contextmgr(
+ *status_codes, body_iter=body_iter,
+ headers=headers, expect_headers=expect_headers,
+ give_connect=capture_conn):
resp = req.get_response(self.ssc)
self.assertEqual(resp.status_int, 201)
@@ -1627,8 +1628,9 @@ class TestServerSideCopyMiddlewareWithEC(unittest.TestCase):
'X-Obj-Multiphase-Commit': 'yes'
}
# TODO possibly use FakeApp here
- with set_http_connect(*status_codes, body_iter=body_iter,
- headers=headers, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(
+ *status_codes, body_iter=body_iter,
+ headers=headers, expect_headers=expect_headers):
resp = req.get_response(self.ssc)
self.assertEqual(resp.status_int, 416)
self.assertEqual(resp.content_length, len(range_not_satisfiable_body))
diff --git a/test/unit/proxy/__init__.py b/test/unit/proxy/__init__.py
index e69de29bb2d1..42a4d83850ab 100644
--- a/test/unit/proxy/__init__.py
+++ b/test/unit/proxy/__init__.py
@@ -0,0 +1,71 @@
+# Copyright (c) 2010-2016 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" Swift tests """
+
+from __future__ import print_function
+from contextlib import contextmanager
+
+from swift.proxy import controllers as proxy_controllers
+
+from test.unit import (FakeMemcache, fake_http_connect)
+
+
+def _make_callback_func(calls):
+ def callback(ipaddr, port, device, partition, method, path,
+ headers=None, query_string=None, ssl=False):
+ context = {}
+ context['method'] = method
+ context['path'] = path
+ context['headers'] = headers or {}
+ calls.append(context)
+ return callback
+
+
+def set_http_connect(*args, **kwargs):
+ new_connect = fake_http_connect(*args, **kwargs)
+ proxy_controllers.base.http_connect = new_connect
+ proxy_controllers.obj.http_connect = new_connect
+ proxy_controllers.account.http_connect = new_connect
+ proxy_controllers.container.http_connect = new_connect
+ return new_connect
+
+
+@contextmanager
+def save_globals():
+ orig_http_connect = getattr(proxy_controllers.base, 'http_connect',
+ None)
+ orig_account_info = getattr(proxy_controllers.Controller,
+ 'account_info', None)
+ orig_container_info = getattr(proxy_controllers.Controller,
+ 'container_info', None)
+
+ try:
+ yield True
+ finally:
+ proxy_controllers.Controller.account_info = orig_account_info
+ proxy_controllers.base.http_connect = orig_http_connect
+ proxy_controllers.obj.http_connect = orig_http_connect
+ proxy_controllers.account.http_connect = orig_http_connect
+ proxy_controllers.container.http_connect = orig_http_connect
+ proxy_controllers.Controller.container_info = orig_container_info
+
+
+class FakeMemcacheReturnsNone(FakeMemcache):
+
+ def get(self, key):
+ # Returns None as the timestamp of the container; assumes we're only
+ # using the FakeMemcache for container existence checks.
+ return None
diff --git a/test/unit/proxy/controllers/test_account.py b/test/unit/proxy/controllers/test_account.py
index a46dcc90f1b3..d9d598cbf03c 100644
--- a/test/unit/proxy/controllers/test_account.py
+++ b/test/unit/proxy/controllers/test_account.py
@@ -15,27 +15,29 @@
import mock
import unittest
+from abc import ABCMeta
+from contextlib import contextmanager
+from six import add_metaclass
from swift.common.swob import Request, Response
-from swift.common.middleware.acl import format_acl
+from swift.common.middleware.acl import (
+ parse_acl, format_acl)
from swift.proxy import server as proxy_server
from swift.proxy.controllers.base import headers_to_account_info
from swift.common import constraints
-from test.unit import fake_http_connect, FakeRing, FakeMemcache
+from test.unit import (
+ fake_http_connect, FakeRing, FakeMemcache, patch_policies)
+from test.unit.proxy import (
+ _make_callback_func, save_globals,
+ set_http_connect, FakeMemcacheReturnsNone)
from swift.common.storage_policy import StoragePolicy
from swift.common.request_helpers import get_sys_meta_prefix
import swift.proxy.controllers.base
from swift.proxy.controllers.base import get_account_info
-from test.unit import patch_policies
-
-@patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())])
-class TestAccountController(unittest.TestCase):
- def setUp(self):
- self.app = proxy_server.Application(
- None, FakeMemcache(),
- account_ring=FakeRing(), container_ring=FakeRing())
+@add_metaclass(ABCMeta)
+class AccountControllerMeta(object):
def _make_callback_func(self, context):
def callback(ipaddr, port, device, partition, method, path,
@@ -62,6 +64,470 @@ class TestAccountController(unittest.TestCase):
'Expected %s but got %s. Failed case: %s' %
(expected, resp.status_int, str(responses)))
+ def assert_status_map(self, method, statuses, expected, env_expected=None,
+ headers=None, **kwargs):
+ headers = headers or {}
+ with save_globals():
+ set_http_connect(*statuses, **kwargs)
+ req = Request.blank('/v1/a', {}, headers=headers)
+ self.app.update_request(req)
+ res = method(req)
+ self.assertEqual(res.status_int, expected)
+ infocache = res.environ.get('swift.infocache', {})
+ if env_expected:
+ self.assertEqual(infocache['account/a']['status'],
+ env_expected)
+ set_http_connect(*statuses)
+ req = Request.blank('/v1/a/', {})
+ self.app.update_request(req)
+ res = method(req)
+ infocache = res.environ.get('swift.infocache', {})
+ self.assertEqual(res.status_int, expected)
+ if env_expected:
+ self.assertEqual(infocache['account/a']['status'],
+ env_expected)
+
+
+@patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())])
+class TestAccountController(AccountControllerMeta, unittest.TestCase):
+ def setUp(self):
+ self.app = proxy_server.Application(
+ None, FakeMemcache(),
+ account_ring=FakeRing(), container_ring=FakeRing())
+
+ def test_OPTIONS(self):
+ with save_globals():
+ self.app.allow_account_management = False
+ controller = proxy_server.AccountController(self.app, 'account')
+ req = Request.blank('/v1/account', {'REQUEST_METHOD': 'OPTIONS'})
+ req.content_length = 0
+ resp = controller.OPTIONS(req)
+ self.assertEqual(200, resp.status_int)
+ for verb in 'OPTIONS GET POST HEAD'.split():
+ self.assertIn(verb, resp.headers['Allow'])
+ self.assertEqual(len(resp.headers['Allow'].split(', ')), 4)
+
+ # Test a CORS OPTIONS request (i.e. including Origin and
+ # Access-Control-Request-Method headers)
+ self.app.allow_account_management = False
+ controller = proxy_server.AccountController(self.app, 'account')
+ req = Request.blank(
+ '/v1/account', {'REQUEST_METHOD': 'OPTIONS'},
+ headers={'Origin': 'http://foo.com',
+ 'Access-Control-Request-Method': 'GET'})
+ req.content_length = 0
+ resp = controller.OPTIONS(req)
+ self.assertEqual(200, resp.status_int)
+ for verb in 'OPTIONS GET POST HEAD'.split():
+ self.assertIn(verb, resp.headers['Allow'])
+ self.assertEqual(len(resp.headers['Allow'].split(', ')), 4)
+
+ self.app.allow_account_management = True
+ controller = proxy_server.AccountController(self.app, 'account')
+ req = Request.blank('/v1/account', {'REQUEST_METHOD': 'OPTIONS'})
+ req.content_length = 0
+ resp = controller.OPTIONS(req)
+ self.assertEqual(200, resp.status_int)
+ for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split():
+ self.assertIn(verb, resp.headers['Allow'])
+ self.assertEqual(len(resp.headers['Allow'].split(', ')), 6)
+
+ def test_GET(self):
+ with save_globals():
+ controller = proxy_server.AccountController(self.app, 'a')
+ # GET returns after the first successful call to an Account Server
+ self.assert_status_map(controller.GET, (200,), 200, 200)
+ self.assert_status_map(controller.GET, (503, 200), 200, 200)
+ self.assert_status_map(controller.GET, (503, 503, 200), 200, 200)
+ self.assert_status_map(controller.GET, (204,), 204, 204)
+ self.assert_status_map(controller.GET, (503, 204), 204, 204)
+ self.assert_status_map(controller.GET, (503, 503, 204), 204, 204)
+ self.assert_status_map(controller.GET, (404, 200), 200, 200)
+ self.assert_status_map(controller.GET, (404, 404, 200), 200, 200)
+ self.assert_status_map(controller.GET, (404, 503, 204), 204, 204)
+ # If Account servers fail, if autocreate = False, return majority
+ # response
+ self.assert_status_map(controller.GET, (404, 404, 404), 404, 404)
+ self.assert_status_map(controller.GET, (404, 404, 503), 404, 404)
+ self.assert_status_map(controller.GET, (404, 503, 503), 503)
+
+ self.app.memcache = FakeMemcacheReturnsNone()
+ self.assert_status_map(controller.GET, (404, 404, 404), 404, 404)
+
+ def test_GET_autocreate(self):
+ with save_globals():
+ controller = proxy_server.AccountController(self.app, 'a')
+ self.app.memcache = FakeMemcacheReturnsNone()
+ self.assertFalse(self.app.account_autocreate)
+ # Repeat the test for autocreate = False and 404 by all
+ self.assert_status_map(controller.GET,
+ (404, 404, 404), 404)
+ self.assert_status_map(controller.GET,
+ (404, 503, 404), 404)
+ # When autocreate is True, if none of the nodes respond 2xx
+ # And quorum of the nodes responded 404,
+ # ALL nodes are asked to create the account
+ # If successful, the GET request is repeated.
+ controller.app.account_autocreate = True
+ self.assert_status_map(controller.GET,
+ (404, 404, 404), 204)
+ self.assert_status_map(controller.GET,
+ (404, 503, 404), 204)
+
+ # We always return 503 if no majority between 4xx, 3xx or 2xx found
+ self.assert_status_map(controller.GET,
+ (500, 500, 400), 503)
+
+ def test_HEAD(self):
+ # Same behaviour as GET
+ with save_globals():
+ controller = proxy_server.AccountController(self.app, 'a')
+ self.assert_status_map(controller.HEAD, (200,), 200, 200)
+ self.assert_status_map(controller.HEAD, (503, 200), 200, 200)
+ self.assert_status_map(controller.HEAD, (503, 503, 200), 200, 200)
+ self.assert_status_map(controller.HEAD, (204,), 204, 204)
+ self.assert_status_map(controller.HEAD, (503, 204), 204, 204)
+ self.assert_status_map(controller.HEAD, (204, 503, 503), 204, 204)
+ self.assert_status_map(controller.HEAD, (204,), 204, 204)
+ self.assert_status_map(controller.HEAD, (404, 404, 404), 404, 404)
+ self.assert_status_map(controller.HEAD, (404, 404, 200), 200, 200)
+ self.assert_status_map(controller.HEAD, (404, 200), 200, 200)
+ self.assert_status_map(controller.HEAD, (404, 404, 503), 404, 404)
+ self.assert_status_map(controller.HEAD, (404, 503, 503), 503)
+ self.assert_status_map(controller.HEAD, (404, 503, 204), 204, 204)
+
+ def test_HEAD_autocreate(self):
+ # Same behaviour as GET
+ with save_globals():
+ controller = proxy_server.AccountController(self.app, 'a')
+ self.app.memcache = FakeMemcacheReturnsNone()
+ self.assertFalse(self.app.account_autocreate)
+ self.assert_status_map(controller.HEAD,
+ (404, 404, 404), 404)
+ controller.app.account_autocreate = True
+ self.assert_status_map(controller.HEAD,
+ (404, 404, 404), 204)
+ self.assert_status_map(controller.HEAD,
+ (500, 404, 404), 204)
+ # We always return 503 if no majority between 4xx, 3xx or 2xx found
+ self.assert_status_map(controller.HEAD,
+ (500, 500, 400), 503)
+
+ def test_POST_autocreate(self):
+ with save_globals():
+ controller = proxy_server.AccountController(self.app, 'a')
+ self.app.memcache = FakeMemcacheReturnsNone()
+ # first test with autocreate being False
+ self.assertFalse(self.app.account_autocreate)
+ self.assert_status_map(controller.POST,
+ (404, 404, 404), 404)
+ # next turn it on and test account being created than updated
+ controller.app.account_autocreate = True
+ self.assert_status_map(
+ controller.POST,
+ (404, 404, 404, 202, 202, 202, 201, 201, 201), 201)
+ # account_info PUT account POST account
+ self.assert_status_map(
+ controller.POST,
+ (404, 404, 503, 201, 201, 503, 204, 204, 504), 204)
+ # what if create fails
+ self.assert_status_map(
+ controller.POST,
+ (404, 404, 404, 403, 403, 403, 400, 400, 400), 400)
+
+ def test_POST_autocreate_with_sysmeta(self):
+ with save_globals():
+ controller = proxy_server.AccountController(self.app, 'a')
+ self.app.memcache = FakeMemcacheReturnsNone()
+ # first test with autocreate being False
+ self.assertFalse(self.app.account_autocreate)
+ self.assert_status_map(controller.POST,
+ (404, 404, 404), 404)
+ # next turn it on and test account being created than updated
+ controller.app.account_autocreate = True
+ calls = []
+ callback = _make_callback_func(calls)
+ key, value = 'X-Account-Sysmeta-Blah', 'something'
+ headers = {key: value}
+ self.assert_status_map(
+ controller.POST,
+ (404, 404, 404, 202, 202, 202, 201, 201, 201), 201,
+ # POST , autocreate PUT, POST again
+ headers=headers,
+ give_connect=callback)
+ self.assertEqual(9, len(calls))
+ for call in calls:
+ self.assertIn(key, call['headers'],
+ '%s call, key %s missing in headers %s' %
+ (call['method'], key, call['headers']))
+ self.assertEqual(value, call['headers'][key])
+
+ def test_connection_refused(self):
+ self.app.account_ring.get_nodes('account')
+ for dev in self.app.account_ring.devs:
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = 1 # can't connect on this port
+ controller = proxy_server.AccountController(self.app, 'account')
+ req = Request.blank('/v1/account', environ={'REQUEST_METHOD': 'HEAD'})
+ self.app.update_request(req)
+ resp = controller.HEAD(req)
+ self.assertEqual(resp.status_int, 503)
+
+ def test_other_socket_error(self):
+ self.app.account_ring.get_nodes('account')
+ for dev in self.app.account_ring.devs:
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = -1 # invalid port number
+ controller = proxy_server.AccountController(self.app, 'account')
+ req = Request.blank('/v1/account', environ={'REQUEST_METHOD': 'HEAD'})
+ self.app.update_request(req)
+ resp = controller.HEAD(req)
+ self.assertEqual(resp.status_int, 503)
+
+ def test_response_get_accept_ranges_header(self):
+ with save_globals():
+ set_http_connect(200, 200, body='{}')
+ controller = proxy_server.AccountController(self.app, 'account')
+ req = Request.blank('/v1/a?format=json')
+ self.app.update_request(req)
+ res = controller.GET(req)
+ self.assertIn('accept-ranges', res.headers)
+ self.assertEqual(res.headers['accept-ranges'], 'bytes')
+
+ def test_response_head_accept_ranges_header(self):
+ with save_globals():
+ set_http_connect(200, 200, body='{}')
+ controller = proxy_server.AccountController(self.app, 'account')
+ req = Request.blank('/v1/a?format=json')
+ self.app.update_request(req)
+ res = controller.HEAD(req)
+ res.body
+ self.assertIn('accept-ranges', res.headers)
+ self.assertEqual(res.headers['accept-ranges'], 'bytes')
+
+ def test_PUT(self):
+ with save_globals():
+ controller = proxy_server.AccountController(self.app, 'account')
+
+ def test_status_map(statuses, expected, **kwargs):
+ set_http_connect(*statuses, **kwargs)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a', {})
+ req.content_length = 0
+ self.app.update_request(req)
+ res = controller.PUT(req)
+ expected = str(expected)
+ self.assertEqual(res.status[:len(expected)], expected)
+ test_status_map((201, 201, 201), 405)
+ self.app.allow_account_management = True
+ test_status_map((201, 201, 201), 201)
+ test_status_map((201, 201, 500), 201)
+ test_status_map((201, 500, 500), 503)
+ test_status_map((204, 500, 404), 503)
+
+ def test_PUT_max_account_name_length(self):
+ with save_globals():
+ self.app.allow_account_management = True
+ limit = constraints.MAX_ACCOUNT_NAME_LENGTH
+ controller = proxy_server.AccountController(self.app, '1' * limit)
+ self.assert_status_map(controller.PUT, (201, 201, 201), 201)
+ controller = proxy_server.AccountController(
+ self.app, '2' * (limit + 1))
+ self.assert_status_map(controller.PUT, (201, 201, 201), 400)
+
+ def test_PUT_connect_exceptions(self):
+ with save_globals():
+ self.app.allow_account_management = True
+ controller = proxy_server.AccountController(self.app, 'account')
+ self.assert_status_map(controller.PUT, (201, 201, -1), 201)
+ self.assert_status_map(controller.PUT, (201, -1, -1), 503)
+ self.assert_status_map(controller.PUT, (503, 503, -1), 503)
+
+ def test_PUT_status(self):
+ with save_globals():
+ self.app.allow_account_management = True
+ controller = proxy_server.AccountController(self.app, 'account')
+ self.assert_status_map(controller.PUT, (201, 201, 202), 202)
+
+ def metadata_helper(self, method):
+ for test_header, test_value in (
+ ('X-Account-Meta-TestHeader', 'TestValue'),
+ ('X-Account-Meta-TestHeader', ''),
+ ('X-Remove-Account-Meta-TestHeader', 'anything')):
+ test_errors = []
+
+ def test_connect(ipaddr, port, device, partition, method, path,
+ headers=None, query_string=None):
+ if path == '/a':
+ find_header = test_header
+ find_value = test_value
+ if find_header.lower().startswith('x-remove-'):
+ find_header = \
+ find_header.lower().replace('-remove', '', 1)
+ find_value = ''
+ for k, v in headers.items():
+ if k.lower() == find_header.lower() and \
+ v == find_value:
+ break
+ else:
+ test_errors.append('%s: %s not in %s' %
+ (find_header, find_value, headers))
+ with save_globals():
+ self.app.allow_account_management = True
+ controller = \
+ proxy_server.AccountController(self.app, 'a')
+ set_http_connect(201, 201, 201, give_connect=test_connect)
+ req = Request.blank('/v1/a/c',
+ environ={'REQUEST_METHOD': method},
+ headers={test_header: test_value})
+ self.app.update_request(req)
+ getattr(controller, method)(req)
+ self.assertEqual(test_errors, [])
+
+ def test_PUT_metadata(self):
+ self.metadata_helper('PUT')
+
+ def test_POST_metadata(self):
+ self.metadata_helper('POST')
+
+ def bad_metadata_helper(self, method):
+ with save_globals():
+ self.app.allow_account_management = True
+ controller = proxy_server.AccountController(self.app, 'a')
+ set_http_connect(200, 201, 201, 201)
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method})
+ self.app.update_request(req)
+ resp = getattr(controller, method)(req)
+ self.assertEqual(resp.status_int, 201)
+
+ set_http_connect(201, 201, 201)
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
+ headers={'X-Account-Meta-' +
+ ('a' * constraints.MAX_META_NAME_LENGTH): 'v'})
+ self.app.update_request(req)
+ resp = getattr(controller, method)(req)
+ self.assertEqual(resp.status_int, 201)
+ set_http_connect(201, 201, 201)
+ req = Request.blank(
+ '/v1/a/c', environ={'REQUEST_METHOD': method},
+ headers={'X-Account-Meta-' +
+ ('a' * (constraints.MAX_META_NAME_LENGTH + 1)): 'v'})
+ self.app.update_request(req)
+ resp = getattr(controller, method)(req)
+ self.assertEqual(resp.status_int, 400)
+
+ set_http_connect(201, 201, 201)
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
+ headers={'X-Account-Meta-Too-Long':
+ 'a' * constraints.MAX_META_VALUE_LENGTH})
+ self.app.update_request(req)
+ resp = getattr(controller, method)(req)
+ self.assertEqual(resp.status_int, 201)
+ set_http_connect(201, 201, 201)
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
+ headers={'X-Account-Meta-Too-Long':
+ 'a' * (constraints.MAX_META_VALUE_LENGTH + 1)})
+ self.app.update_request(req)
+ resp = getattr(controller, method)(req)
+ self.assertEqual(resp.status_int, 400)
+
+ set_http_connect(201, 201, 201)
+ headers = {}
+ for x in range(constraints.MAX_META_COUNT):
+ headers['X-Account-Meta-%d' % x] = 'v'
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
+ headers=headers)
+ self.app.update_request(req)
+ resp = getattr(controller, method)(req)
+ self.assertEqual(resp.status_int, 201)
+ set_http_connect(201, 201, 201)
+ headers = {}
+ for x in range(constraints.MAX_META_COUNT + 1):
+ headers['X-Account-Meta-%d' % x] = 'v'
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
+ headers=headers)
+ self.app.update_request(req)
+ resp = getattr(controller, method)(req)
+ self.assertEqual(resp.status_int, 400)
+
+ set_http_connect(201, 201, 201)
+ headers = {}
+ header_value = 'a' * constraints.MAX_META_VALUE_LENGTH
+ size = 0
+ x = 0
+ while size < (constraints.MAX_META_OVERALL_SIZE - 4
+ - constraints.MAX_META_VALUE_LENGTH):
+ size += 4 + constraints.MAX_META_VALUE_LENGTH
+ headers['X-Account-Meta-%04d' % x] = header_value
+ x += 1
+ if constraints.MAX_META_OVERALL_SIZE - size > 1:
+ headers['X-Account-Meta-a'] = \
+ 'a' * (constraints.MAX_META_OVERALL_SIZE - size - 1)
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
+ headers=headers)
+ self.app.update_request(req)
+ resp = getattr(controller, method)(req)
+ self.assertEqual(resp.status_int, 201)
+ set_http_connect(201, 201, 201)
+ headers['X-Account-Meta-a'] = \
+ 'a' * (constraints.MAX_META_OVERALL_SIZE - size)
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
+ headers=headers)
+ self.app.update_request(req)
+ resp = getattr(controller, method)(req)
+ self.assertEqual(resp.status_int, 400)
+
+ def test_PUT_bad_metadata(self):
+ self.bad_metadata_helper('PUT')
+
+ def test_POST_bad_metadata(self):
+ self.bad_metadata_helper('POST')
+
+ def test_DELETE(self):
+ with save_globals():
+ controller = proxy_server.AccountController(self.app, 'account')
+
+ def test_status_map(statuses, expected, **kwargs):
+ set_http_connect(*statuses, **kwargs)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a', {'REQUEST_METHOD': 'DELETE'})
+ req.content_length = 0
+ self.app.update_request(req)
+ res = controller.DELETE(req)
+ expected = str(expected)
+ self.assertEqual(res.status[:len(expected)], expected)
+ test_status_map((201, 201, 201), 405)
+ self.app.allow_account_management = True
+ test_status_map((201, 201, 201), 201)
+ test_status_map((201, 201, 500), 201)
+ test_status_map((201, 500, 500), 503)
+ test_status_map((204, 500, 404), 503)
+
+ def test_DELETE_with_query_string(self):
+ # Extra safety in case someone typos a query string for an
+ # account-level DELETE request that was really meant to be caught by
+ # some middleware.
+ with save_globals():
+ controller = proxy_server.AccountController(self.app, 'account')
+
+ def test_status_map(statuses, expected, **kwargs):
+ set_http_connect(*statuses, **kwargs)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a?whoops',
+ environ={'REQUEST_METHOD': 'DELETE'})
+ req.content_length = 0
+ self.app.update_request(req)
+ res = controller.DELETE(req)
+ expected = str(expected)
+ self.assertEqual(res.status[:len(expected)], expected)
+ test_status_map((201, 201, 201), 400)
+ self.app.allow_account_management = True
+ test_status_map((201, 201, 201), 400)
+ test_status_map((201, 201, 500), 400)
+ test_status_map((201, 500, 500), 400)
+ test_status_map((204, 500, 404), 400)
+
+ # Original functions start from here
def test_account_info_in_response_env(self):
controller = proxy_server.AccountController(self.app, 'AUTH_bob')
with mock.patch('swift.proxy.controllers.base.http_connect',
@@ -316,7 +782,7 @@ class TestAccountController(unittest.TestCase):
@patch_policies(
[StoragePolicy(0, 'zero', True, object_ring=FakeRing(replicas=4))])
-class TestAccountController4Replicas(TestAccountController):
+class TestAccountController4Replicas(AccountControllerMeta, unittest.TestCase):
def setUp(self):
self.app = proxy_server.Application(
None,
@@ -402,5 +868,231 @@ class TestGetAccountInfo(unittest.TestCase):
self.assertEqual(410, info.get('status'))
+@patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())])
+class TestAccountControllerFakeGetResponse(unittest.TestCase):
+ """
+ Test all the faked-out GET responses for accounts that don't exist. They
+ have to match the responses for empty accounts that really exist.
+ """
+ def setUp(self):
+ conf = {'account_autocreate': 'yes'}
+ self.app = proxy_server.Application(conf, FakeMemcache(),
+ account_ring=FakeRing(),
+ container_ring=FakeRing())
+ self.app.memcache = FakeMemcacheReturnsNone()
+
+ def test_GET_autocreate_accept_json(self):
+ with save_globals():
+ set_http_connect(*([404] * 100)) # nonexistent: all backends 404
+ req = Request.blank(
+ '/v1/a', headers={'Accept': 'application/json'},
+ environ={'REQUEST_METHOD': 'GET',
+ 'PATH_INFO': '/v1/a'})
+ resp = req.get_response(self.app)
+ self.assertEqual(200, resp.status_int)
+ self.assertEqual('application/json; charset=utf-8',
+ resp.headers['Content-Type'])
+ self.assertEqual("[]", resp.body)
+
+ def test_GET_autocreate_format_json(self):
+ with save_globals():
+ set_http_connect(*([404] * 100)) # nonexistent: all backends 404
+ req = Request.blank('/v1/a?format=json',
+ environ={'REQUEST_METHOD': 'GET',
+ 'PATH_INFO': '/v1/a',
+ 'QUERY_STRING': 'format=json'})
+ resp = req.get_response(self.app)
+ self.assertEqual(200, resp.status_int)
+ self.assertEqual('application/json; charset=utf-8',
+ resp.headers['Content-Type'])
+ self.assertEqual("[]", resp.body)
+
+ def test_GET_autocreate_accept_xml(self):
+ with save_globals():
+ set_http_connect(*([404] * 100)) # nonexistent: all backends 404
+ req = Request.blank('/v1/a', headers={"Accept": "text/xml"},
+ environ={'REQUEST_METHOD': 'GET',
+ 'PATH_INFO': '/v1/a'})
+
+ resp = req.get_response(self.app)
+ self.assertEqual(200, resp.status_int)
+
+ self.assertEqual('text/xml; charset=utf-8',
+ resp.headers['Content-Type'])
+ empty_xml_listing = ('<?xml version="1.0" encoding="UTF-8"?>\n'
+ '<account name="a">\n</account>')
+ self.assertEqual(empty_xml_listing, resp.body)
+
+ def test_GET_autocreate_format_xml(self):
+ with save_globals():
+ set_http_connect(*([404] * 100)) # nonexistent: all backends 404
+ req = Request.blank('/v1/a?format=xml',
+ environ={'REQUEST_METHOD': 'GET',
+ 'PATH_INFO': '/v1/a',
+ 'QUERY_STRING': 'format=xml'})
+ resp = req.get_response(self.app)
+ self.assertEqual(200, resp.status_int)
+ self.assertEqual('application/xml; charset=utf-8',
+ resp.headers['Content-Type'])
+ empty_xml_listing = ('<?xml version="1.0" encoding="UTF-8"?>\n'
+ '<account name="a">\n</account>')
+ self.assertEqual(empty_xml_listing, resp.body)
+
+ def test_GET_autocreate_accept_unknown(self):
+ with save_globals():
+ set_http_connect(*([404] * 100)) # nonexistent: all backends 404
+ req = Request.blank('/v1/a', headers={"Accept": "mystery/meat"},
+ environ={'REQUEST_METHOD': 'GET',
+ 'PATH_INFO': '/v1/a'})
+ resp = req.get_response(self.app)
+ self.assertEqual(406, resp.status_int)
+
+ def test_GET_autocreate_format_invalid_utf8(self):
+ with save_globals():
+ set_http_connect(*([404] * 100)) # nonexistent: all backends 404
+ req = Request.blank('/v1/a?format=\xff\xfe',
+ environ={'REQUEST_METHOD': 'GET',
+ 'PATH_INFO': '/v1/a',
+ 'QUERY_STRING': 'format=\xff\xfe'})
+ resp = req.get_response(self.app)
+ self.assertEqual(400, resp.status_int)
+
+ def test_account_acl_header_access(self):
+ acl = {
+ 'admin': ['AUTH_alice'],
+ 'read-write': ['AUTH_bob'],
+ 'read-only': ['AUTH_carol'],
+ }
+ prefix = get_sys_meta_prefix('account')
+ privileged_headers = {(prefix + 'core-access-control'): format_acl(
+ version=2, acl_dict=acl)}
+
+ app = proxy_server.Application(
+ None, FakeMemcache(), account_ring=FakeRing(),
+ container_ring=FakeRing())
+
+ with save_globals():
+ # Mock account server will provide privileged information (ACLs)
+ set_http_connect(200, 200, 200, headers=privileged_headers)
+ req = Request.blank('/v1/a', environ={'REQUEST_METHOD': 'GET'})
+ resp = app.handle_request(req)
+
+ # Not a swift_owner -- ACLs should NOT be in response
+ header = 'X-Account-Access-Control'
+ self.assertNotIn(header, resp.headers, '%r was in %r' % (
+ header, resp.headers))
+
+ # Same setup -- mock acct server will provide ACLs
+ set_http_connect(200, 200, 200, headers=privileged_headers)
+ req = Request.blank('/v1/a', environ={'REQUEST_METHOD': 'GET',
+ 'swift_owner': True})
+ resp = app.handle_request(req)
+
+ # For a swift_owner, the ACLs *should* be in response
+ self.assertIn(header, resp.headers, '%r not in %r' % (
+ header, resp.headers))
+
+ def test_account_acls_through_delegation(self):
+
+ # Define a way to grab the requests sent out from the AccountController
+ # to the Account Server, and a way to inject responses we'd like the
+ # Account Server to return.
+ resps_to_send = []
+
+ @contextmanager
+ def patch_account_controller_method(verb):
+ old_method = getattr(proxy_server.AccountController, verb)
+ new_method = lambda self, req, *_, **__: resps_to_send.pop(0)
+ try:
+ setattr(proxy_server.AccountController, verb, new_method)
+ yield
+ finally:
+ setattr(proxy_server.AccountController, verb, old_method)
+
+ def make_test_request(http_method, swift_owner=True):
+ env = {
+ 'REQUEST_METHOD': http_method,
+ 'swift_owner': swift_owner,
+ }
+ acl = {
+ 'admin': ['foo'],
+ 'read-write': ['bar'],
+ 'read-only': ['bas'],
+ }
+ headers = {} if http_method in ('GET', 'HEAD') else {
+ 'x-account-access-control': format_acl(version=2, acl_dict=acl)
+ }
+
+ return Request.blank('/v1/a', environ=env, headers=headers)
+
+ # Our AccountController will invoke methods to communicate with the
+ # Account Server, and they will return responses like these:
+ def make_canned_response(http_method):
+ acl = {
+ 'admin': ['foo'],
+ 'read-write': ['bar'],
+ 'read-only': ['bas'],
+ }
+ headers = {'x-account-sysmeta-core-access-control': format_acl(
+ version=2, acl_dict=acl)}
+ canned_resp = Response(headers=headers)
+ canned_resp.environ = {
+ 'PATH_INFO': '/acct',
+ 'REQUEST_METHOD': http_method,
+ }
+ resps_to_send.append(canned_resp)
+
+ app = proxy_server.Application(
+ None, FakeMemcache(), account_ring=FakeRing(),
+ container_ring=FakeRing())
+ app.allow_account_management = True
+
+ ext_header = 'x-account-access-control'
+ with patch_account_controller_method('GETorHEAD_base'):
+ # GET/HEAD requests should remap sysmeta headers from acct server
+ for verb in ('GET', 'HEAD'):
+ make_canned_response(verb)
+ req = make_test_request(verb)
+ resp = app.handle_request(req)
+ h = parse_acl(version=2, data=resp.headers.get(ext_header))
+ self.assertEqual(h['admin'], ['foo'])
+ self.assertEqual(h['read-write'], ['bar'])
+ self.assertEqual(h['read-only'], ['bas'])
+
+ # swift_owner = False: GET/HEAD shouldn't return sensitive info
+ make_canned_response(verb)
+ req = make_test_request(verb, swift_owner=False)
+ resp = app.handle_request(req)
+ h = resp.headers
+ self.assertIsNone(h.get(ext_header))
+
+ # swift_owner unset: GET/HEAD shouldn't return sensitive info
+ make_canned_response(verb)
+ req = make_test_request(verb, swift_owner=False)
+ del req.environ['swift_owner']
+ resp = app.handle_request(req)
+ h = resp.headers
+ self.assertIsNone(h.get(ext_header))
+
+ # Verify that PUT/POST requests remap sysmeta headers from acct server
+ with patch_account_controller_method('make_requests'):
+ make_canned_response('PUT')
+ req = make_test_request('PUT')
+ resp = app.handle_request(req)
+
+ h = parse_acl(version=2, data=resp.headers.get(ext_header))
+ self.assertEqual(h['admin'], ['foo'])
+ self.assertEqual(h['read-write'], ['bar'])
+ self.assertEqual(h['read-only'], ['bas'])
+
+ make_canned_response('POST')
+ req = make_test_request('POST')
+ resp = app.handle_request(req)
+
+ h = parse_acl(version=2, data=resp.headers.get(ext_header))
+ self.assertEqual(h['admin'], ['foo'])
+ self.assertEqual(h['read-write'], ['bar'])
+ self.assertEqual(h['read-only'], ['bas'])
+
if __name__ == '__main__':
unittest.main()
diff --git a/test/unit/proxy/controllers/test_container.py b/test/unit/proxy/controllers/test_container.py
index 1cc0d1986fd4..239ccd76592d 100644
--- a/test/unit/proxy/controllers/test_container.py
+++ b/test/unit/proxy/controllers/test_container.py
@@ -15,35 +15,54 @@
import mock
import unittest
-
+from abc import ABCMeta
from eventlet import Timeout
+import random
+import re
+from contextlib import contextmanager
+from six import add_metaclass
+import time
+from swift.common import constraints
from swift.common.swob import Request
from swift.proxy import server as proxy_server
-from swift.proxy.controllers.base import headers_to_container_info
+import swift.proxy.controllers
+from swift.proxy.controllers import base as proxy_base
+from swift.proxy.controllers.base import (
+ cors_validation, headers_to_container_info)
from test.unit import fake_http_connect, FakeRing, FakeMemcache
-from swift.common.storage_policy import StoragePolicy
+from swift.common.exceptions import ChunkReadTimeout
from swift.common.request_helpers import get_sys_meta_prefix
+from swift.common.storage_policy import StoragePolicy, POLICIES
+from swift.common.swob import (
+ HTTPException, HTTPUnauthorized, Response)
-from test.unit import patch_policies, mocked_http_conn, debug_logger
+from test.unit import (
+ patch_policies, mocked_http_conn, debug_logger)
from test.unit.common.ring.test_ring import TestRingBase
-from test.unit.proxy.test_server import node_error_count
+from test.unit.proxy import (
+ _make_callback_func, save_globals,
+ set_http_connect, FakeMemcacheReturnsNone)
+from test.unit.proxy.test_server import (
+ node_error_count, node_last_error,
+ set_node_errors, sortHeaderNames)
-@patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())])
-class TestContainerController(TestRingBase):
+@add_metaclass(ABCMeta)
+class ContainerControllerMeta(object):
CONTAINER_REPLICAS = 3
def setUp(self):
TestRingBase.setUp(self)
- self.logger = debug_logger()
- self.container_ring = FakeRing(replicas=self.CONTAINER_REPLICAS,
- max_more_nodes=9)
- self.app = proxy_server.Application(None, FakeMemcache(),
- logger=self.logger,
- account_ring=FakeRing(),
- container_ring=self.container_ring)
+ self.container_ring = FakeRing(
+ replicas=self.CONTAINER_REPLICAS,
+ max_more_nodes=9)
+ self.app = proxy_server.Application(
+ None, FakeMemcache(),
+ account_ring=FakeRing(),
+ container_ring=self.container_ring,
+ logger=debug_logger())
self.account_info = {
'status': 200,
@@ -95,6 +114,10 @@ class TestContainerController(TestRingBase):
'Expected %s but got %s. Failed case: %s' %
(expected, resp.status_int, str(responses)))
+
+@patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())])
+class TestContainerController(ContainerControllerMeta, TestRingBase):
+
def test_container_info_got_cached(self):
controller = proxy_server.ContainerController(self.app, 'a', 'c')
with mock.patch('swift.proxy.controllers.base.http_connect',
@@ -305,9 +328,1270 @@ class TestContainerController(TestRingBase):
self._assert_responses('POST', POST_TEST_CASES)
+@patch_policies([
+ StoragePolicy(0, 'zero', True, object_ring=FakeRing(base_port=3000)),
+ StoragePolicy(1, 'one', False, object_ring=FakeRing(base_port=3000)),
+ StoragePolicy(2, 'two', False, True, object_ring=FakeRing(base_port=3000))
+])
+class TestContainerControllerBasePort(unittest.TestCase):
+ def setUp(self):
+ self.app = proxy_server.Application(
+ None, FakeMemcache(),
+ account_ring=FakeRing(),
+ container_ring=FakeRing(base_port=2000),
+ logger=debug_logger())
+
+ def test_convert_policy_to_index(self):
+ controller = swift.proxy.controllers.ContainerController(self.app,
+ 'a', 'c')
+ expected = {
+ 'zero': 0,
+ 'ZeRo': 0,
+ 'one': 1,
+ 'OnE': 1,
+ }
+ for name, index in expected.items():
+ req = Request.blank('/a/c', headers={'Content-Length': '0',
+ 'Content-Type': 'text/plain',
+ 'X-Storage-Policy': name})
+ self.assertEqual(controller._convert_policy_to_index(req), index)
+ # default test
+ req = Request.blank('/a/c', headers={'Content-Length': '0',
+ 'Content-Type': 'text/plain'})
+ self.assertIsNone(controller._convert_policy_to_index(req))
+ # negative test
+ req = Request.blank('/a/c',
+ headers={'Content-Length': '0',
+ 'Content-Type': 'text/plain',
+ 'X-Storage-Policy': 'nada'})
+ self.assertRaises(HTTPException, controller._convert_policy_to_index,
+ req)
+ # storage policy two is deprecated
+ req = Request.blank('/a/c', headers={'Content-Length': '0',
+ 'Content-Type': 'text/plain',
+ 'X-Storage-Policy': 'two'})
+ self.assertRaises(HTTPException, controller._convert_policy_to_index,
+ req)
+
+ def test_convert_index_to_name(self):
+ policy = random.choice(list(POLICIES))
+ req = Request.blank('/v1/a/c')
+ with mocked_http_conn(
+ 200, 200,
+ headers={'X-Backend-Storage-Policy-Index': int(policy)},
+ ) as fake_conn:
+ resp = req.get_response(self.app)
+ self.assertRaises(StopIteration, fake_conn.code_iter.next)
+ self.assertEqual(resp.status_int, 200)
+ self.assertEqual(resp.headers['X-Storage-Policy'], policy.name)
+
+ def test_no_convert_index_to_name_when_container_not_found(self):
+ policy = random.choice(list(POLICIES))
+ req = Request.blank('/v1/a/c')
+ with mocked_http_conn(
+ 200, 404, 404, 404,
+ headers={'X-Backend-Storage-Policy-Index':
+ int(policy)}) as fake_conn:
+ resp = req.get_response(self.app)
+ self.assertRaises(StopIteration, fake_conn.code_iter.next)
+ self.assertEqual(resp.status_int, 404)
+ self.assertIsNone(resp.headers['X-Storage-Policy'])
+
+ def test_error_convert_index_to_name(self):
+ req = Request.blank('/v1/a/c')
+ with mocked_http_conn(
+ 200, 200,
+ headers={'X-Backend-Storage-Policy-Index': '-1'}) as fake_conn:
+ resp = req.get_response(self.app)
+ self.assertRaises(StopIteration, fake_conn.code_iter.next)
+ self.assertEqual(resp.status_int, 200)
+ self.assertIsNone(resp.headers['X-Storage-Policy'])
+ error_lines = self.app.logger.get_lines_for_level('error')
+ self.assertEqual(2, len(error_lines))
+ for msg in error_lines:
+ expected = "Could not translate " \
+ "X-Backend-Storage-Policy-Index ('-1')"
+ self.assertIn(expected, msg)
+
+ def test_transfer_headers(self):
+ src_headers = {'x-remove-versions-location': 'x',
+ 'x-container-read': '*:user',
+ 'x-remove-container-sync-key': 'x'}
+ dst_headers = {'x-versions-location': 'backup'}
+ controller = swift.proxy.controllers.ContainerController(self.app,
+ 'a', 'c')
+ controller.transfer_headers(src_headers, dst_headers)
+ expected_headers = {'x-versions-location': '',
+ 'x-container-read': '*:user',
+ 'x-container-sync-key': ''}
+ self.assertEqual(dst_headers, expected_headers)
+
+ def assert_status_map(self, method, statuses, expected,
+ raise_exc=False, missing_container=False):
+ with save_globals():
+ kwargs = {}
+ if raise_exc:
+ kwargs['raise_exc'] = raise_exc
+ kwargs['missing_container'] = missing_container
+ set_http_connect(*statuses, **kwargs)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a/c', headers={'Content-Length': '0',
+ 'Content-Type': 'text/plain'})
+ self.app.update_request(req)
+ res = method(req)
+ self.assertEqual(res.status_int, expected)
+ set_http_connect(*statuses, **kwargs)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a/c/', headers={'Content-Length': '0',
+ 'Content-Type': 'text/plain'})
+ self.app.update_request(req)
+ res = method(req)
+ self.assertEqual(res.status_int, expected)
+
+ def test_HEAD_GET(self):
+ with save_globals():
+ controller = proxy_server.ContainerController(self.app, 'a', 'c')
+
+ def test_status_map(statuses, expected,
+ c_expected=None, a_expected=None, **kwargs):
+ set_http_connect(*statuses, **kwargs)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a/c', {})
+ self.app.update_request(req)
+ res = controller.HEAD(req)
+ self.assertEqual(res.status[:len(str(expected))],
+ str(expected))
+ infocache = res.environ.get('swift.infocache', {})
+ if expected < 400:
+ self.assertIn('x-works', res.headers)
+ self.assertEqual(res.headers['x-works'], 'yes')
+ if expected < 300:
+ self.assertIn('last-modified', res.headers)
+ self.assertEqual(res.headers['last-modified'], '1')
+ if c_expected:
+ self.assertIn('container/a/c', infocache)
+ self.assertEqual(
+ infocache['container/a/c']['status'],
+ c_expected)
+ else:
+ self.assertNotIn('container/a/c', infocache)
+ if a_expected:
+ self.assertIn('account/a', infocache)
+ self.assertEqual(infocache['account/a']['status'],
+ a_expected)
+ else:
+ self.assertNotIn('account/a', res.environ)
+
+ set_http_connect(*statuses, **kwargs)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a/c', {})
+ self.app.update_request(req)
+ res = controller.GET(req)
+ self.assertEqual(res.status[:len(str(expected))],
+ str(expected))
+ infocache = res.environ.get('swift.infocache', {})
+ if expected < 400:
+ self.assertIn('x-works', res.headers)
+ self.assertEqual(res.headers['x-works'], 'yes')
+ if expected < 300:
+ self.assertIn('last-modified', res.headers)
+ self.assertEqual(res.headers['last-modified'], '1')
+ if c_expected:
+ self.assertIn('container/a/c', infocache)
+ self.assertEqual(
+ infocache['container/a/c']['status'],
+ c_expected)
+ else:
+ self.assertNotIn('container/a/c', infocache)
+ if a_expected:
+ self.assertIn('account/a', infocache)
+ self.assertEqual(infocache['account/a']['status'],
+ a_expected)
+ else:
+ self.assertNotIn('account/a', infocache)
+ # In all the following tests cache 200 for account
+ # return and cache vary for container
+ # return 200 and cache 200 for account and container
+ test_status_map((200, 200, 404, 404), 200, 200, 200)
+ test_status_map((200, 200, 500, 404), 200, 200, 200)
+ # return 304 don't cache container
+ test_status_map((200, 304, 500, 404), 304, None, 200)
+ # return 404 and cache 404 for container
+ test_status_map((200, 404, 404, 404), 404, 404, 200)
+ test_status_map((200, 404, 404, 500), 404, 404, 200)
+ # return 503, don't cache container
+ test_status_map((200, 500, 500, 500), 503, None, 200)
+ self.assertFalse(self.app.account_autocreate)
+
+ # return 404 (as account is not found) and don't cache container
+ test_status_map((404, 404, 404), 404, None, 404)
+
+ # cache a 204 for the account because it's sort of like it
+ # exists
+ self.app.account_autocreate = True
+ test_status_map((404, 404, 404), 404, None, 204)
+
+ def test_PUT_policy_headers(self):
+ backend_requests = []
+
+ def capture_requests(ipaddr, port, device, partition, method,
+ path, headers=None, query_string=None):
+ if method == 'PUT':
+ backend_requests.append(headers)
+
+ def test_policy(requested_policy):
+ with save_globals():
+ mock_conn = set_http_connect(200, 201, 201, 201,
+ give_connect=capture_requests)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a/test', method='PUT',
+ headers={'Content-Length': 0})
+ if requested_policy:
+ expected_policy = requested_policy
+ req.headers['X-Storage-Policy'] = policy.name
+ else:
+ expected_policy = POLICIES.default
+ res = req.get_response(self.app)
+ if expected_policy.is_deprecated:
+ self.assertEqual(res.status_int, 400)
+ self.assertEqual(0, len(backend_requests))
+ expected = 'is deprecated'
+ self.assertIn(expected, res.body,
+ '%r did not include %r' % (
+ res.body, expected))
+ return
+ self.assertEqual(res.status_int, 201)
+ self.assertEqual(
+ expected_policy.object_ring.replicas,
+ len(backend_requests))
+ for headers in backend_requests:
+ if not requested_policy:
+ self.assertNotIn('X-Backend-Storage-Policy-Index',
+ headers)
+ self.assertIn('X-Backend-Storage-Policy-Default',
+ headers)
+ self.assertEqual(
+ int(expected_policy),
+ int(headers['X-Backend-Storage-Policy-Default']))
+ else:
+ self.assertIn('X-Backend-Storage-Policy-Index',
+ headers)
+ self.assertEqual(int(headers
+ ['X-Backend-Storage-Policy-Index']),
+ int(policy))
+ # make sure all mocked responses are consumed
+ self.assertRaises(StopIteration, mock_conn.code_iter.next)
+
+ test_policy(None) # no policy header
+ for policy in POLICIES:
+ backend_requests = [] # reset backend requests
+ test_policy(policy)
+
+ def test_PUT(self):
+ with save_globals():
+ controller = proxy_server.ContainerController(self.app, 'account',
+ 'container')
+
+ def test_status_map(statuses, expected, **kwargs):
+ set_http_connect(*statuses, **kwargs)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a/c', {})
+ req.content_length = 0
+ self.app.update_request(req)
+ res = controller.PUT(req)
+ expected = str(expected)
+ self.assertEqual(res.status[:len(expected)], expected)
+
+ test_status_map((200, 201, 201, 201), 201, missing_container=True)
+ test_status_map((200, 201, 201, 500), 201, missing_container=True)
+ test_status_map((200, 204, 404, 404), 404, missing_container=True)
+ test_status_map((200, 204, 500, 404), 503, missing_container=True)
+ self.assertFalse(self.app.account_autocreate)
+ test_status_map((404, 404, 404), 404, missing_container=True)
+ self.app.account_autocreate = True
+ # fail to retrieve account info
+ test_status_map(
+ (503, 503, 503), # account_info fails on 503
+ 404, missing_container=True)
+ # account fail after creation
+ test_status_map(
+ (404, 404, 404, # account_info fails on 404
+ 201, 201, 201, # PUT account
+ 404, 404, 404), # account_info fail
+ 404, missing_container=True)
+ test_status_map(
+ (503, 503, 404, # account_info fails on 404
+ 503, 503, 503, # PUT account
+ 503, 503, 404), # account_info fail
+ 404, missing_container=True)
+ # put fails
+ test_status_map(
+ (404, 404, 404, # account_info fails on 404
+ 201, 201, 201, # PUT account
+ 200, # account_info success
+ 503, 503, 201), # put container fail
+ 503, missing_container=True)
+ # all goes according to plan
+ test_status_map(
+ (404, 404, 404, # account_info fails on 404
+ 201, 201, 201, # PUT account
+ 200, # account_info success
+ 201, 201, 201), # put container success
+ 201, missing_container=True)
+ test_status_map(
+ (503, 404, 404, # account_info fails on 404
+ 503, 201, 201, # PUT account
+ 503, 200, # account_info success
+ 503, 201, 201), # put container success
+ 201, missing_container=True)
+
+ def test_PUT_autocreate_account_with_sysmeta(self):
+ # x-account-sysmeta headers in a container PUT request should be
+ # transferred to the account autocreate PUT request
+ with save_globals():
+ controller = proxy_server.ContainerController(self.app, 'account',
+ 'container')
+
+ def test_status_map(statuses, expected, headers=None, **kwargs):
+ set_http_connect(*statuses, **kwargs)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a/c', {}, headers=headers)
+ req.content_length = 0
+ self.app.update_request(req)
+ res = controller.PUT(req)
+ expected = str(expected)
+ self.assertEqual(res.status[:len(expected)], expected)
+
+ self.app.account_autocreate = True
+ calls = []
+ callback = _make_callback_func(calls)
+ key, value = 'X-Account-Sysmeta-Blah', 'something'
+ headers = {key: value}
+
+ # all goes according to plan
+ test_status_map(
+ (404, 404, 404, # account_info fails on 404
+ 201, 201, 201, # PUT account
+ 200, # account_info success
+ 201, 201, 201), # put container success
+ 201, missing_container=True,
+ headers=headers,
+ give_connect=callback)
+
+ self.assertEqual(10, len(calls))
+ for call in calls[3:6]:
+ self.assertEqual('/account', call['path'])
+ self.assertIn(key, call['headers'],
+ '%s call, key %s missing in headers %s' % (
+ call['method'], key, call['headers']))
+ self.assertEqual(value, call['headers'][key])
+
+ def test_POST(self):
+ with save_globals():
+ controller = proxy_server.ContainerController(self.app, 'account',
+ 'container')
+
+ def test_status_map(statuses, expected, **kwargs):
+ set_http_connect(*statuses, **kwargs)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a/c', {})
+ req.content_length = 0
+ self.app.update_request(req)
+ res = controller.POST(req)
+ expected = str(expected)
+ self.assertEqual(res.status[:len(expected)], expected)
+
+ test_status_map((200, 201, 201, 201), 201, missing_container=True)
+ test_status_map((200, 201, 201, 500), 201, missing_container=True)
+ test_status_map((200, 204, 404, 404), 404, missing_container=True)
+ test_status_map((200, 204, 500, 404), 503, missing_container=True)
+ self.assertFalse(self.app.account_autocreate)
+ test_status_map((404, 404, 404), 404, missing_container=True)
+ self.app.account_autocreate = True
+ test_status_map((404, 404, 404), 404, missing_container=True)
+
+ def test_PUT_max_containers_per_account(self):
+ with save_globals():
+ self.app.max_containers_per_account = 12346
+ controller = proxy_server.ContainerController(self.app, 'account',
+ 'container')
+ self.assert_status_map(controller.PUT,
+ (200, 201, 201, 201), 201,
+ missing_container=True)
+
+ self.app.max_containers_per_account = 12345
+ controller = proxy_server.ContainerController(self.app, 'account',
+ 'container')
+ self.assert_status_map(controller.PUT,
+ (200, 200, 201, 201, 201), 201,
+ missing_container=True)
+
+ controller = proxy_server.ContainerController(self.app, 'account',
+ 'container_new')
+
+ self.assert_status_map(controller.PUT, (200, 404, 404, 404), 403,
+ missing_container=True)
+
+ self.app.max_containers_per_account = 12345
+ self.app.max_containers_whitelist = ['account']
+ controller = proxy_server.ContainerController(self.app, 'account',
+ 'container')
+ self.assert_status_map(controller.PUT,
+ (200, 201, 201, 201), 201,
+ missing_container=True)
+
+ def test_PUT_max_container_name_length(self):
+ with save_globals():
+ limit = constraints.MAX_CONTAINER_NAME_LENGTH
+ controller = proxy_server.ContainerController(self.app, 'account',
+ '1' * limit)
+ self.assert_status_map(controller.PUT,
+ (200, 201, 201, 201), 201,
+ missing_container=True)
+ controller = proxy_server.ContainerController(self.app, 'account',
+ '2' * (limit + 1))
+ self.assert_status_map(controller.PUT, (201, 201, 201), 400,
+ missing_container=True)
+
+ def test_PUT_connect_exceptions(self):
+ with save_globals():
+ controller = proxy_server.ContainerController(self.app, 'account',
+ 'container')
+ self.assert_status_map(controller.PUT, (200, 201, 201, -1), 201,
+ missing_container=True)
+ self.assert_status_map(controller.PUT, (200, 201, -1, -1), 503,
+ missing_container=True)
+ self.assert_status_map(controller.PUT, (200, 503, 503, -1), 503,
+ missing_container=True)
+
+ def test_acc_missing_returns_404(self):
+ for meth in ('DELETE', 'PUT'):
+ with save_globals():
+ self.app.memcache = FakeMemcacheReturnsNone()
+ self.app._error_limiting = {}
+ controller = proxy_server.ContainerController(self.app,
+ 'account',
+ 'container')
+ if meth == 'PUT':
+ set_http_connect(200, 200, 200, 200, 200, 200,
+ missing_container=True)
+ else:
+ set_http_connect(200, 200, 200, 200)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a/c',
+ environ={'REQUEST_METHOD': meth})
+ self.app.update_request(req)
+ resp = getattr(controller, meth)(req)
+ self.assertEqual(resp.status_int, 200)
+
+ set_http_connect(404, 404, 404, 200, 200, 200)
+ # Make sure it is a blank request wthout env caching
+ req = Request.blank('/v1/a/c',
+ environ={'REQUEST_METHOD': meth})
+ resp = getattr(controller, meth)(req)
+ self.assertEqual(resp.status_int, 404)
+
+ set_http_connect(503, 404, 404)
+ # Make sure it is a blank request wthout env caching
+ req = Request.blank('/v1/a/c',
+ environ={'REQUEST_METHOD': meth})
+ resp = getattr(controller, meth)(req)
+ self.assertEqual(resp.status_int, 404)
+
+ set_http_connect(503, 404, raise_exc=True)
+ # Make sure it is a blank request wthout env caching
+ req = Request.blank('/v1/a/c',
+ environ={'REQUEST_METHOD': meth})
+ resp = getattr(controller, meth)(req)
+ self.assertEqual(resp.status_int, 404)
+
+ for dev in self.app.account_ring.devs:
+ set_node_errors(self.app, dev,
+ self.app.error_suppression_limit + 1,
+ time.time())
+ set_http_connect(200, 200, 200, 200, 200, 200)
+ # Make sure it is a blank request wthout env caching
+ req = Request.blank('/v1/a/c',
+ environ={'REQUEST_METHOD': meth})
+ resp = getattr(controller, meth)(req)
+ self.assertEqual(resp.status_int, 404)
+
+ def test_put_locking(self):
+
+ class MockMemcache(FakeMemcache):
+
+ def __init__(self, allow_lock=None):
+ self.allow_lock = allow_lock
+ super(MockMemcache, self).__init__()
+
+ @contextmanager
+ def soft_lock(self, key, timeout=0, retries=5):
+ if self.allow_lock:
+ yield True
+ else:
+ raise NotImplementedError
+
+ with save_globals():
+ controller = proxy_server.ContainerController(self.app, 'account',
+ 'container')
+ self.app.memcache = MockMemcache(allow_lock=True)
+ set_http_connect(200, 201, 201, 201,
+ missing_container=True)
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'PUT'})
+ self.app.update_request(req)
+ res = controller.PUT(req)
+ self.assertEqual(res.status_int, 201)
+
+ def test_error_limiting(self):
+ with save_globals():
+ controller = proxy_server.ContainerController(self.app, 'account',
+ 'container')
+ container_ring = controller.app.container_ring
+ controller.app.sort_nodes = lambda l: l
+ self.assert_status_map(controller.HEAD, (200, 503, 200, 200), 200,
+ missing_container=False)
+
+ self.assertEqual(
+ node_error_count(controller.app, container_ring.devs[0]), 2)
+ self.assertTrue(
+ node_last_error(controller.app, container_ring.devs[0])
+ is not None)
+ for _junk in range(self.app.error_suppression_limit):
+ self.assert_status_map(controller.HEAD,
+ (200, 503, 503, 503), 503)
+ self.assertEqual(
+ node_error_count(controller.app, container_ring.devs[0]),
+ self.app.error_suppression_limit + 1)
+ self.assert_status_map(controller.HEAD, (200, 200, 200, 200), 503)
+ self.assertTrue(
+ node_last_error(controller.app, container_ring.devs[0])
+ is not None)
+ self.assert_status_map(controller.PUT, (200, 201, 201, 201), 503,
+ missing_container=True)
+ self.assert_status_map(controller.DELETE,
+ (200, 204, 204, 204), 503)
+ self.app.error_suppression_interval = -300
+ self.assert_status_map(controller.HEAD, (200, 200, 200, 200), 200)
+ self.assert_status_map(controller.DELETE, (200, 204, 204, 204),
+ 404, raise_exc=True)
+
+ def test_DELETE(self):
+ with save_globals():
+ controller = proxy_server.ContainerController(self.app, 'account',
+ 'container')
+ self.assert_status_map(controller.DELETE,
+ (200, 204, 204, 204), 204)
+ self.assert_status_map(controller.DELETE,
+ (200, 204, 204, 503), 204)
+ self.assert_status_map(controller.DELETE,
+ (200, 204, 503, 503), 503)
+ self.assert_status_map(controller.DELETE,
+ (200, 204, 404, 404), 404)
+ self.assert_status_map(controller.DELETE,
+ (200, 404, 404, 404), 404)
+ self.assert_status_map(controller.DELETE,
+ (200, 204, 503, 404), 503)
+
+ self.app.memcache = FakeMemcacheReturnsNone()
+ # 200: Account check, 404x3: Container check
+ self.assert_status_map(controller.DELETE,
+ (200, 404, 404, 404), 404)
+
+ def test_response_get_accept_ranges_header(self):
+ with save_globals():
+ set_http_connect(200, 200, body='{}')
+ controller = proxy_server.ContainerController(self.app, 'account',
+ 'container')
+ req = Request.blank('/v1/a/c?format=json')
+ self.app.update_request(req)
+ res = controller.GET(req)
+ self.assertIn('accept-ranges', res.headers)
+ self.assertEqual(res.headers['accept-ranges'], 'bytes')
+
+ def test_response_head_accept_ranges_header(self):
+ with save_globals():
+ set_http_connect(200, 200, body='{}')
+ controller = proxy_server.ContainerController(self.app, 'account',
+ 'container')
+ req = Request.blank('/v1/a/c?format=json')
+ self.app.update_request(req)
+ res = controller.HEAD(req)
+ self.assertIn('accept-ranges', res.headers)
+ self.assertEqual(res.headers['accept-ranges'], 'bytes')
+
+ def test_PUT_metadata(self):
+ self.metadata_helper('PUT')
+
+ def test_POST_metadata(self):
+ self.metadata_helper('POST')
+
+ def metadata_helper(self, method):
+ for test_header, test_value in (
+ ('X-Container-Meta-TestHeader', 'TestValue'),
+ ('X-Container-Meta-TestHeader', ''),
+ ('X-Remove-Container-Meta-TestHeader', 'anything'),
+ ('X-Container-Read', '.r:*'),
+ ('X-Remove-Container-Read', 'anything'),
+ ('X-Container-Write', 'anyone'),
+ ('X-Remove-Container-Write', 'anything')):
+ test_errors = []
+
+ def test_connect(ipaddr, port, device, partition, method, path,
+ headers=None, query_string=None):
+ if path == '/a/c':
+ find_header = test_header
+ find_value = test_value
+ if find_header.lower().startswith('x-remove-'):
+ find_header = \
+ find_header.lower().replace('-remove', '', 1)
+ find_value = ''
+ for k, v in headers.items():
+ if k.lower() == find_header.lower() and \
+ v == find_value:
+ break
+ else:
+ test_errors.append('%s: %s not in %s' %
+ (find_header, find_value, headers))
+ with save_globals():
+ controller = \
+ proxy_server.ContainerController(self.app, 'a', 'c')
+ set_http_connect(200, 201, 201, 201, give_connect=test_connect)
+ req = Request.blank(
+ '/v1/a/c',
+ environ={'REQUEST_METHOD': method, 'swift_owner': True},
+ headers={test_header: test_value})
+ self.app.update_request(req)
+ getattr(controller, method)(req)
+ self.assertEqual(test_errors, [])
+
+ def test_PUT_bad_metadata(self):
+ self.bad_metadata_helper('PUT')
+
+ def test_POST_bad_metadata(self):
+ self.bad_metadata_helper('POST')
+
+ def bad_metadata_helper(self, method):
+ with save_globals():
+ controller = proxy_server.ContainerController(self.app, 'a', 'c')
+ set_http_connect(200, 201, 201, 201)
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method})
+ self.app.update_request(req)
+ resp = getattr(controller, method)(req)
+ self.assertEqual(resp.status_int, 201)
+
+ set_http_connect(201, 201, 201)
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
+ headers={'X-Container-Meta-' +
+ ('a' * constraints.MAX_META_NAME_LENGTH): 'v'})
+ self.app.update_request(req)
+ resp = getattr(controller, method)(req)
+ self.assertEqual(resp.status_int, 201)
+ set_http_connect(201, 201, 201)
+ req = Request.blank(
+ '/v1/a/c', environ={'REQUEST_METHOD': method},
+ headers={'X-Container-Meta-' +
+ ('a' * (constraints.MAX_META_NAME_LENGTH + 1)): 'v'})
+ self.app.update_request(req)
+ resp = getattr(controller, method)(req)
+ self.assertEqual(resp.status_int, 400)
+
+ set_http_connect(201, 201, 201)
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
+ headers={'X-Container-Meta-Too-Long':
+ 'a' * constraints.MAX_META_VALUE_LENGTH})
+ self.app.update_request(req)
+ resp = getattr(controller, method)(req)
+ self.assertEqual(resp.status_int, 201)
+ set_http_connect(201, 201, 201)
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
+ headers={'X-Container-Meta-Too-Long':
+ 'a' * (constraints.MAX_META_VALUE_LENGTH + 1)})
+ self.app.update_request(req)
+ resp = getattr(controller, method)(req)
+ self.assertEqual(resp.status_int, 400)
+
+ set_http_connect(201, 201, 201)
+ headers = {}
+ for x in range(constraints.MAX_META_COUNT):
+ headers['X-Container-Meta-%d' % x] = 'v'
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
+ headers=headers)
+ self.app.update_request(req)
+ resp = getattr(controller, method)(req)
+ self.assertEqual(resp.status_int, 201)
+ set_http_connect(201, 201, 201)
+ headers = {}
+ for x in range(constraints.MAX_META_COUNT + 1):
+ headers['X-Container-Meta-%d' % x] = 'v'
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
+ headers=headers)
+ self.app.update_request(req)
+ resp = getattr(controller, method)(req)
+ self.assertEqual(resp.status_int, 400)
+
+ set_http_connect(201, 201, 201)
+ headers = {}
+ header_value = 'a' * constraints.MAX_META_VALUE_LENGTH
+ size = 0
+ x = 0
+ while size < (constraints.MAX_META_OVERALL_SIZE - 4
+ - constraints.MAX_META_VALUE_LENGTH):
+ size += 4 + constraints.MAX_META_VALUE_LENGTH
+ headers['X-Container-Meta-%04d' % x] = header_value
+ x += 1
+ if constraints.MAX_META_OVERALL_SIZE - size > 1:
+ headers['X-Container-Meta-a'] = \
+ 'a' * (constraints.MAX_META_OVERALL_SIZE - size - 1)
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
+ headers=headers)
+ self.app.update_request(req)
+ resp = getattr(controller, method)(req)
+ self.assertEqual(resp.status_int, 201)
+ set_http_connect(201, 201, 201)
+ headers['X-Container-Meta-a'] = \
+ 'a' * (constraints.MAX_META_OVERALL_SIZE - size)
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
+ headers=headers)
+ self.app.update_request(req)
+ resp = getattr(controller, method)(req)
+ self.assertEqual(resp.status_int, 400)
+
+ def test_POST_calls_clean_acl(self):
+ called = [False]
+
+ def clean_acl(header, value):
+ called[0] = True
+ raise ValueError('fake error')
+ with save_globals():
+ set_http_connect(200, 201, 201, 201)
+ controller = proxy_server.ContainerController(self.app, 'account',
+ 'container')
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'POST'},
+ headers={'X-Container-Read': '.r:*'})
+ req.environ['swift.clean_acl'] = clean_acl
+ self.app.update_request(req)
+ controller.POST(req)
+ self.assertTrue(called[0])
+ called[0] = False
+ with save_globals():
+ set_http_connect(200, 201, 201, 201)
+ controller = proxy_server.ContainerController(self.app, 'account',
+ 'container')
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'POST'},
+ headers={'X-Container-Write': '.r:*'})
+ req.environ['swift.clean_acl'] = clean_acl
+ self.app.update_request(req)
+ controller.POST(req)
+ self.assertTrue(called[0])
+
+ def test_PUT_calls_clean_acl(self):
+ called = [False]
+
+ def clean_acl(header, value):
+ called[0] = True
+ raise ValueError('fake error')
+ with save_globals():
+ set_http_connect(200, 201, 201, 201)
+ controller = proxy_server.ContainerController(self.app, 'account',
+ 'container')
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'PUT'},
+ headers={'X-Container-Read': '.r:*'})
+ req.environ['swift.clean_acl'] = clean_acl
+ self.app.update_request(req)
+ controller.PUT(req)
+ self.assertTrue(called[0])
+ called[0] = False
+ with save_globals():
+ set_http_connect(200, 201, 201, 201)
+ controller = proxy_server.ContainerController(self.app, 'account',
+ 'container')
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'PUT'},
+ headers={'X-Container-Write': '.r:*'})
+ req.environ['swift.clean_acl'] = clean_acl
+ self.app.update_request(req)
+ controller.PUT(req)
+ self.assertTrue(called[0])
+
+ def test_GET_no_content(self):
+ with save_globals():
+ set_http_connect(200, 204, 204, 204)
+ controller = proxy_server.ContainerController(self.app, 'a', 'c')
+ req = Request.blank('/v1/a/c')
+ self.app.update_request(req)
+ res = controller.GET(req)
+ self.assertEqual(res.status_int, 204)
+ ic = res.environ['swift.infocache']
+ self.assertEqual(ic['container/a/c']['status'], 204)
+ self.assertEqual(res.content_length, 0)
+ self.assertNotIn('transfer-encoding', res.headers)
+
+ def test_GET_calls_authorize(self):
+ called = [False]
+
+ def authorize(req):
+ called[0] = True
+ return HTTPUnauthorized(request=req)
+ with save_globals():
+ set_http_connect(200, 201, 201, 201)
+ controller = proxy_server.ContainerController(self.app, 'a', 'c')
+ req = Request.blank('/v1/a/c')
+ req.environ['swift.authorize'] = authorize
+ self.app.update_request(req)
+ res = controller.GET(req)
+ self.assertEqual(
+ res.environ['swift.infocache']['container/a/c']['status'],
+ 201)
+ self.assertTrue(called[0])
+
+ def test_HEAD_calls_authorize(self):
+ called = [False]
+
+ def authorize(req):
+ called[0] = True
+ return HTTPUnauthorized(request=req)
+ with save_globals():
+ set_http_connect(200, 201, 201, 201)
+ controller = proxy_server.ContainerController(self.app, 'a', 'c')
+ req = Request.blank('/v1/a/c', {'REQUEST_METHOD': 'HEAD'})
+ req.environ['swift.authorize'] = authorize
+ self.app.update_request(req)
+ controller.HEAD(req)
+ self.assertTrue(called[0])
+
+ def test_unauthorized_requests_when_account_not_found(self):
+ # verify unauthorized container requests always return response
+ # from swift.authorize
+ called = [0, 0]
+
+ def authorize(req):
+ called[0] += 1
+ return HTTPUnauthorized(request=req)
+
+ def account_info(*args):
+ called[1] += 1
+ return None, None, None
+
+ def _do_test(method):
+ with save_globals():
+ swift.proxy.controllers.Controller.account_info = account_info
+ app = proxy_server.Application(None, FakeMemcache(),
+ account_ring=FakeRing(),
+ container_ring=FakeRing())
+ set_http_connect(201, 201, 201)
+ req = Request.blank('/v1/a/c', {'REQUEST_METHOD': method})
+ req.environ['swift.authorize'] = authorize
+ self.app.update_request(req)
+ res = app.handle_request(req)
+ return res
+
+ for method in ('PUT', 'POST', 'DELETE'):
+ # no delay_denial on method, expect one call to authorize
+ called = [0, 0]
+ res = _do_test(method)
+ self.assertEqual(401, res.status_int)
+ self.assertEqual([1, 0], called)
+
+ for method in ('HEAD', 'GET'):
+ # delay_denial on method, expect two calls to authorize
+ called = [0, 0]
+ res = _do_test(method)
+ self.assertEqual(401, res.status_int)
+ self.assertEqual([2, 1], called)
+
+ def test_authorized_requests_when_account_not_found(self):
+ # verify authorized container requests always return 404 when
+ # account not found
+ called = [0, 0]
+
+ def authorize(req):
+ called[0] += 1
+
+ def account_info(*args):
+ called[1] += 1
+ return None, None, None
+
+ def _do_test(method):
+ with save_globals():
+ swift.proxy.controllers.Controller.account_info = account_info
+ app = proxy_server.Application(None, FakeMemcache(),
+ account_ring=FakeRing(),
+ container_ring=FakeRing())
+ set_http_connect(201, 201, 201)
+ req = Request.blank('/v1/a/c', {'REQUEST_METHOD': method})
+ req.environ['swift.authorize'] = authorize
+ self.app.update_request(req)
+ res = app.handle_request(req)
+ return res
+
+ for method in ('PUT', 'POST', 'DELETE', 'HEAD', 'GET'):
+ # expect one call to authorize
+ called = [0, 0]
+ res = _do_test(method)
+ self.assertEqual(404, res.status_int)
+ self.assertEqual([1, 1], called)
+
+ def test_OPTIONS_get_info_drops_origin(self):
+ with save_globals():
+ controller = proxy_server.ContainerController(self.app, 'a', 'c')
+
+ count = [0]
+
+ def my_get_info(app, env, account, container=None,
+ ret_not_found=False, swift_source=None):
+ if count[0] > 11:
+ return {}
+ count[0] += 1
+ if not container:
+ return {'some': 'stuff'}
+ return proxy_base.was_get_info(
+ app, env, account, container, ret_not_found, swift_source)
+
+ proxy_base.was_get_info = proxy_base.get_info
+ with mock.patch.object(proxy_base, 'get_info', my_get_info):
+ proxy_base.get_info = my_get_info
+ req = Request.blank(
+ '/v1/a/c',
+ {'REQUEST_METHOD': 'OPTIONS'},
+ headers={'Origin': 'http://foo.com',
+ 'Access-Control-Request-Method': 'GET'})
+ controller.OPTIONS(req)
+ self.assertLess(count[0], 11)
+
+ def test_OPTIONS(self):
+ with save_globals():
+ controller = proxy_server.ContainerController(self.app, 'a', 'c')
+
+ def my_empty_container_info(*args):
+ return {}
+ controller.container_info = my_empty_container_info
+ req = Request.blank(
+ '/v1/a/c',
+ {'REQUEST_METHOD': 'OPTIONS'},
+ headers={'Origin': 'http://foo.com',
+ 'Access-Control-Request-Method': 'GET'})
+ resp = controller.OPTIONS(req)
+ self.assertEqual(401, resp.status_int)
+
+ def my_empty_origin_container_info(*args):
+ return {'cors': {'allow_origin': None}}
+ controller.container_info = my_empty_origin_container_info
+ req = Request.blank(
+ '/v1/a/c',
+ {'REQUEST_METHOD': 'OPTIONS'},
+ headers={'Origin': 'http://foo.com',
+ 'Access-Control-Request-Method': 'GET'})
+ resp = controller.OPTIONS(req)
+ self.assertEqual(401, resp.status_int)
+
+ def my_container_info(*args):
+ return {
+ 'cors': {
+ 'allow_origin': 'http://foo.bar:8080 https://foo.bar',
+ 'max_age': '999',
+ }
+ }
+ controller.container_info = my_container_info
+ req = Request.blank(
+ '/v1/a/c',
+ {'REQUEST_METHOD': 'OPTIONS'},
+ headers={'Origin': 'https://foo.bar',
+ 'Access-Control-Request-Method': 'GET'})
+ req.content_length = 0
+ resp = controller.OPTIONS(req)
+ self.assertEqual(200, resp.status_int)
+ self.assertEqual(
+ 'https://foo.bar',
+ resp.headers['access-control-allow-origin'])
+ for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split():
+ self.assertIn(verb,
+ resp.headers['access-control-allow-methods'])
+ self.assertEqual(
+ len(resp.headers['access-control-allow-methods'].split(', ')),
+ 6)
+ self.assertEqual('999', resp.headers['access-control-max-age'])
+ req = Request.blank(
+ '/v1/a/c',
+ {'REQUEST_METHOD': 'OPTIONS'},
+ headers={'Origin': 'https://foo.bar'})
+ req.content_length = 0
+ resp = controller.OPTIONS(req)
+ self.assertEqual(401, resp.status_int)
+ req = Request.blank('/v1/a/c', {'REQUEST_METHOD': 'OPTIONS'})
+ req.content_length = 0
+ resp = controller.OPTIONS(req)
+ self.assertEqual(200, resp.status_int)
+ for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split():
+ self.assertIn(verb, resp.headers['Allow'])
+ self.assertEqual(len(resp.headers['Allow'].split(', ')), 6)
+ req = Request.blank(
+ '/v1/a/c',
+ {'REQUEST_METHOD': 'OPTIONS'},
+ headers={'Origin': 'http://foo.bar',
+ 'Access-Control-Request-Method': 'GET'})
+ resp = controller.OPTIONS(req)
+ self.assertEqual(401, resp.status_int)
+ req = Request.blank(
+ '/v1/a/c',
+ {'REQUEST_METHOD': 'OPTIONS'},
+ headers={'Origin': 'http://foo.bar',
+ 'Access-Control-Request-Method': 'GET'})
+ controller.app.cors_allow_origin = ['http://foo.bar', ]
+ resp = controller.OPTIONS(req)
+ self.assertEqual(200, resp.status_int)
+
+ def my_container_info_wildcard(*args):
+ return {
+ 'cors': {
+ 'allow_origin': '*',
+ 'max_age': '999',
+ }
+ }
+ controller.container_info = my_container_info_wildcard
+ req = Request.blank(
+ '/v1/a/c/o.jpg',
+ {'REQUEST_METHOD': 'OPTIONS'},
+ headers={'Origin': 'https://bar.baz',
+ 'Access-Control-Request-Method': 'GET'})
+ req.content_length = 0
+ resp = controller.OPTIONS(req)
+ self.assertEqual(200, resp.status_int)
+ self.assertEqual('*', resp.headers['access-control-allow-origin'])
+ for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split():
+ self.assertIn(verb,
+ resp.headers['access-control-allow-methods'])
+ self.assertEqual(
+ len(resp.headers['access-control-allow-methods'].split(', ')),
+ 6)
+ self.assertEqual('999', resp.headers['access-control-max-age'])
+
+ req = Request.blank(
+ '/v1/a/c/o.jpg',
+ {'REQUEST_METHOD': 'OPTIONS'},
+ headers={'Origin': 'https://bar.baz',
+ 'Access-Control-Request-Headers':
+ 'x-foo, x-bar, x-auth-token',
+ 'Access-Control-Request-Method': 'GET'}
+ )
+ req.content_length = 0
+ resp = controller.OPTIONS(req)
+ self.assertEqual(200, resp.status_int)
+ self.assertEqual(
+ sortHeaderNames('x-foo, x-bar, x-auth-token'),
+ sortHeaderNames(resp.headers['access-control-allow-headers']))
+
+ def test_CORS_valid(self):
+ with save_globals():
+ controller = proxy_server.ContainerController(self.app, 'a', 'c')
+
+ def stubContainerInfo(*args):
+ return {
+ 'cors': {
+ 'allow_origin': 'http://foo.bar'
+ }
+ }
+ controller.container_info = stubContainerInfo
+
+ def containerGET(controller, req):
+ return Response(headers={
+ 'X-Container-Meta-Color': 'red',
+ 'X-Super-Secret': 'hush',
+ })
+
+ req = Request.blank(
+ '/v1/a/c',
+ {'REQUEST_METHOD': 'GET'},
+ headers={'Origin': 'http://foo.bar'})
+
+ resp = cors_validation(containerGET)(controller, req)
+
+ self.assertEqual(200, resp.status_int)
+ self.assertEqual('http://foo.bar',
+ resp.headers['access-control-allow-origin'])
+ self.assertEqual('red', resp.headers['x-container-meta-color'])
+ # X-Super-Secret is in the response, but not "exposed"
+ self.assertEqual('hush', resp.headers['x-super-secret'])
+ self.assertIn('access-control-expose-headers', resp.headers)
+ exposed = set(
+ h.strip() for h in
+ resp.headers['access-control-expose-headers'].split(','))
+ expected_exposed = set([
+ 'cache-control', 'content-language', 'content-type', 'expires',
+ 'last-modified', 'pragma', 'etag', 'x-timestamp', 'x-trans-id',
+ 'x-openstack-request-id', 'x-container-meta-color'])
+ self.assertEqual(expected_exposed, exposed)
+
+ def _gather_x_account_headers(self, controller_call, req, *connect_args,
+ **kwargs):
+ seen_headers = []
+ to_capture = ('X-Account-Partition', 'X-Account-Host',
+ 'X-Account-Device')
+
+ def capture_headers(ipaddr, port, device, partition, method,
+ path, headers=None, query_string=None):
+ captured = {}
+ for header in to_capture:
+ captured[header] = headers.get(header)
+ seen_headers.append(captured)
+
+ with save_globals():
+ self.app.allow_account_management = True
+
+ set_http_connect(*connect_args, give_connect=capture_headers,
+ **kwargs)
+ resp = controller_call(req)
+ self.assertEqual(2, resp.status_int // 100) # sanity check
+
+ # don't care about the account HEAD, so throw away the
+ # first element
+ return sorted(seen_headers[1:],
+ key=lambda d: d['X-Account-Host'] or 'Z')
+
+ def test_PUT_x_account_headers_with_fewer_account_replicas(self):
+ self.app.account_ring.set_replicas(2)
+ req = Request.blank('/v1/a/c', headers={'': ''})
+ controller = proxy_server.ContainerController(self.app, 'a', 'c')
+
+ seen_headers = self._gather_x_account_headers(
+ controller.PUT, req,
+ 200, 201, 201, 201) # HEAD PUT PUT PUT
+ self.assertEqual(seen_headers, [
+ {'X-Account-Host': '10.0.0.0:1000',
+ 'X-Account-Partition': '0',
+ 'X-Account-Device': 'sda'},
+ {'X-Account-Host': '10.0.0.1:1001',
+ 'X-Account-Partition': '0',
+ 'X-Account-Device': 'sdb'},
+ {'X-Account-Host': None,
+ 'X-Account-Partition': None,
+ 'X-Account-Device': None}
+ ])
+
+ def test_PUT_x_account_headers_with_more_account_replicas(self):
+ self.app.account_ring.set_replicas(4)
+ req = Request.blank('/v1/a/c', headers={'': ''})
+ controller = proxy_server.ContainerController(self.app, 'a', 'c')
+
+ seen_headers = self._gather_x_account_headers(
+ controller.PUT, req,
+ 200, 201, 201, 201) # HEAD PUT PUT PUT
+ self.assertEqual(seen_headers, [
+ {'X-Account-Host': '10.0.0.0:1000,10.0.0.3:1003',
+ 'X-Account-Partition': '0',
+ 'X-Account-Device': 'sda,sdd'},
+ {'X-Account-Host': '10.0.0.1:1001',
+ 'X-Account-Partition': '0',
+ 'X-Account-Device': 'sdb'},
+ {'X-Account-Host': '10.0.0.2:1002',
+ 'X-Account-Partition': '0',
+ 'X-Account-Device': 'sdc'}
+ ])
+
+ def test_DELETE_x_account_headers_with_fewer_account_replicas(self):
+ self.app.account_ring.set_replicas(2)
+ req = Request.blank('/v1/a/c', headers={'': ''})
+ controller = proxy_server.ContainerController(self.app, 'a', 'c')
+
+ seen_headers = self._gather_x_account_headers(
+ controller.DELETE, req,
+ 200, 204, 204, 204) # HEAD DELETE DELETE DELETE
+ self.assertEqual(seen_headers, [
+ {'X-Account-Host': '10.0.0.0:1000',
+ 'X-Account-Partition': '0',
+ 'X-Account-Device': 'sda'},
+ {'X-Account-Host': '10.0.0.1:1001',
+ 'X-Account-Partition': '0',
+ 'X-Account-Device': 'sdb'},
+ {'X-Account-Host': None,
+ 'X-Account-Partition': None,
+ 'X-Account-Device': None}
+ ])
+
+ def test_DELETE_x_account_headers_with_more_account_replicas(self):
+ self.app.account_ring.set_replicas(4)
+ req = Request.blank('/v1/a/c', headers={'': ''})
+ controller = proxy_server.ContainerController(self.app, 'a', 'c')
+
+ seen_headers = self._gather_x_account_headers(
+ controller.DELETE, req,
+ 200, 204, 204, 204) # HEAD DELETE DELETE DELETE
+ self.assertEqual(seen_headers, [
+ {'X-Account-Host': '10.0.0.0:1000,10.0.0.3:1003',
+ 'X-Account-Partition': '0',
+ 'X-Account-Device': 'sda,sdd'},
+ {'X-Account-Host': '10.0.0.1:1001',
+ 'X-Account-Partition': '0',
+ 'X-Account-Device': 'sdb'},
+ {'X-Account-Host': '10.0.0.2:1002',
+ 'X-Account-Partition': '0',
+ 'X-Account-Device': 'sdc'}
+ ])
+
+ def test_PUT_backed_x_timestamp_header(self):
+ timestamps = []
+
+ def capture_timestamps(*args, **kwargs):
+ headers = kwargs['headers']
+ timestamps.append(headers.get('X-Timestamp'))
+
+ req = Request.blank('/v1/a/c', method='PUT', headers={'': ''})
+ with save_globals():
+ new_connect = set_http_connect(200, # account existence check
+ 201, 201, 201,
+ give_connect=capture_timestamps)
+ resp = self.app.handle_request(req)
+
+ # sanity
+ self.assertRaises(StopIteration, new_connect.code_iter.next)
+ self.assertEqual(2, resp.status_int // 100)
+
+ timestamps.pop(0) # account existence check
+ self.assertEqual(3, len(timestamps))
+ for timestamp in timestamps:
+ self.assertEqual(timestamp, timestamps[0])
+ self.assertTrue(re.match('[0-9]{10}\.[0-9]{5}', timestamp))
+
+ def test_DELETE_backed_x_timestamp_header(self):
+ timestamps = []
+
+ def capture_timestamps(*args, **kwargs):
+ headers = kwargs['headers']
+ timestamps.append(headers.get('X-Timestamp'))
+
+ req = Request.blank('/v1/a/c', method='DELETE', headers={'': ''})
+ self.app.update_request(req)
+ with save_globals():
+ new_connect = set_http_connect(200, # account existence check
+ 201, 201, 201,
+ give_connect=capture_timestamps)
+ resp = self.app.handle_request(req)
+
+ # sanity
+ self.assertRaises(StopIteration, new_connect.code_iter.next)
+ self.assertEqual(2, resp.status_int // 100)
+
+ timestamps.pop(0) # account existence check
+ self.assertEqual(3, len(timestamps))
+ for timestamp in timestamps:
+ self.assertEqual(timestamp, timestamps[0])
+ self.assertTrue(re.match('[0-9]{10}\.[0-9]{5}', timestamp))
+
+ def test_node_read_timeout_retry_to_container(self):
+ with save_globals():
+ req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'})
+ self.app.node_timeout = 0.1
+ set_http_connect(200, 200, 200, body='abcdef', slow=[1.0, 1.0])
+ resp = req.get_response(self.app)
+ got_exc = False
+ try:
+ resp.body
+ except ChunkReadTimeout:
+ got_exc = True
+ self.assertTrue(got_exc)
+
+
@patch_policies(
[StoragePolicy(0, 'zero', True, object_ring=FakeRing(replicas=4))])
-class TestContainerController4Replicas(TestContainerController):
+class TestContainerController4Replicas(ContainerControllerMeta, TestRingBase):
CONTAINER_REPLICAS = 4
diff --git a/test/unit/proxy/controllers/test_obj.py b/test/unit/proxy/controllers/test_obj.py
index 4ceccb61b0ae..8d3586e0988e 100644
--- a/test/unit/proxy/controllers/test_obj.py
+++ b/test/unit/proxy/controllers/test_obj.py
@@ -17,8 +17,10 @@
import email.parser
import itertools
import math
+import os
import random
import time
+from tempfile import mkdtemp
import unittest
from collections import defaultdict
from contextlib import contextmanager
@@ -26,26 +28,54 @@ import json
from hashlib import md5
import mock
-from eventlet import Timeout
+from eventlet import Timeout, sleep
from six import BytesIO
from six.moves import range
+from shutil import rmtree
import swift
-from swift.common import utils, swob, exceptions
-from swift.common.exceptions import ChunkWriteTimeout
+from swift.common import constraints, exceptions, swob, utils
+from swift.common.exceptions import ChunkReadTimeout, ChunkWriteTimeout
from swift.common.header_key_dict import HeaderKeyDict
from swift.common.utils import Timestamp
-from swift.obj import server
+from swift.obj import server as object_server
from swift.proxy import server as proxy_server
-from swift.proxy.controllers import obj
+from swift.proxy.controllers.obj import (
+ ECObjectController, ReplicatedObjectController)
from swift.proxy.controllers.base import \
- get_container_info as _real_get_container_info
+ get_container_info as _real_get_container_info, cors_validation
from swift.common.storage_policy import POLICIES, ECDriverError, StoragePolicy
+from swift.common.swob import (
+ Request, Response, HTTPUnauthorized, HTTPException)
-from test.unit import FakeRing, FakeMemcache, fake_http_connect, \
- debug_logger, patch_policies, SlowBody, FakeStatus, \
- encode_frag_archive_bodies
-from test.unit.proxy.test_server import node_error_count
+from test.unit import (
+ FakeLogger, FakeRing, FakeMemcache,
+ fake_http_connect, debug_logger, patch_policies, SlowBody,
+ FakeStatus, encode_frag_archive_bodies, mocked_http_conn)
+from test.unit.proxy.test_server import (
+ node_error_count, node_last_error, set_node_errors)
+from test.unit.proxy import (
+ save_globals, set_http_connect, FakeMemcacheReturnsNone)
+
+from test.unit.helpers import setup_servers
+
+
+STATIC_TIME = time.time()
+_test_context = _test_servers = None
+
+
+def do_setup(object_server):
+ """
+ Duplicated in proxy/test_server.py
+ Can this be refactered?
+ """
+ global _test_context, _test_servers
+ _test_context = setup_servers(object_server)
+ _test_servers = _test_context["test_servers"]
+
+
+def setup():
+ do_setup(object_server)
def unchunk_body(chunked_body):
@@ -60,7 +90,7 @@ def unchunk_body(chunked_body):
@contextmanager
-def set_http_connect(*args, **kwargs):
+def set_http_connect_contextmgr(*args, **kwargs):
old_connect = swift.proxy.controllers.base.http_connect
new_connect = fake_http_connect(*args, **kwargs)
try:
@@ -281,7 +311,7 @@ class BaseObjectControllerMixin(object):
self.app, 'a', 'c', 'o')
req = swift.common.swob.Request.blank('/v1/a/c/o')
self.app.conn_timeout = 0.05
- with set_http_connect(slow_connect=True):
+ with set_http_connect_contextmgr(slow_connect=True):
nodes = [dict(ip='', port='', device='')]
res = controller._connect_put_node(nodes, '', req, {}, ('', ''))
self.assertIsNone(res)
@@ -289,7 +319,7 @@ class BaseObjectControllerMixin(object):
def test_DELETE_simple(self):
req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE')
codes = [204] * self.replicas()
- with set_http_connect(*codes):
+ with set_http_connect_contextmgr(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 204)
@@ -301,7 +331,7 @@ class BaseObjectControllerMixin(object):
req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE')
codes = [404] + [204] * (self.replicas() - 1)
random.shuffle(codes)
- with set_http_connect(*codes):
+ with set_http_connect_contextmgr(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 204)
@@ -312,7 +342,7 @@ class BaseObjectControllerMixin(object):
return
req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE')
codes = [404] * (self.replicas() - 1) + [204]
- with set_http_connect(*codes):
+ with set_http_connect_contextmgr(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 404)
@@ -321,7 +351,7 @@ class BaseObjectControllerMixin(object):
mostly_204s = [204] * self.quorum()
codes = mostly_204s + [404] * (self.replicas() - len(mostly_204s))
self.assertEqual(len(codes), self.replicas())
- with set_http_connect(*codes):
+ with set_http_connect_contextmgr(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 204)
@@ -330,7 +360,7 @@ class BaseObjectControllerMixin(object):
mostly_404s = [404] * self.quorum()
codes = mostly_404s + [204] * (self.replicas() - len(mostly_404s))
self.assertEqual(len(codes), self.replicas())
- with set_http_connect(*codes):
+ with set_http_connect_contextmgr(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 404)
@@ -338,7 +368,7 @@ class BaseObjectControllerMixin(object):
self.obj_ring.set_replicas(4)
req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE')
- with set_http_connect(404, 204, 404, 204):
+ with set_http_connect_contextmgr(404, 204, 404, 204):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 204)
@@ -352,8 +382,8 @@ class BaseObjectControllerMixin(object):
headers = [{}, {}, {'Pick-Me': 'yes'}, {'Pick-Me': 'yes'}]
req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE')
- with set_http_connect(*status_codes, body_iter=bodies,
- headers=headers):
+ with set_http_connect_contextmgr(*status_codes, body_iter=bodies,
+ headers=headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 204)
self.assertEqual(resp.headers.get('Pick-Me'), 'yes')
@@ -362,7 +392,7 @@ class BaseObjectControllerMixin(object):
def test_DELETE_handoff(self):
req = swift.common.swob.Request.blank('/v1/a/c/o', method='DELETE')
codes = [204] * self.replicas()
- with set_http_connect(507, *codes):
+ with set_http_connect_contextmgr(507, *codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 204)
@@ -380,7 +410,7 @@ class BaseObjectControllerMixin(object):
req = swob.Request.blank('/v1/a/c/o', method='PUT', body='',
headers={'Content-Type': 'foo/bar',
'X-Delete-After': t})
- with set_http_connect():
+ with set_http_connect_contextmgr():
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 400)
self.assertEqual('Non-integer X-Delete-After', resp.body)
@@ -397,7 +427,7 @@ class BaseObjectControllerMixin(object):
req = swob.Request.blank('/v1/a/c/o', method='PUT', body='',
headers={'Content-Type': 'foo/bar',
'X-Delete-After': '-60'})
- with set_http_connect():
+ with set_http_connect_contextmgr():
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 400)
self.assertEqual('X-Delete-After in past', resp.body)
@@ -416,7 +446,7 @@ class BaseObjectControllerMixin(object):
req = swob.Request.blank('/v1/a/c/o', method='PUT', body='',
headers={'Content-Type': 'foo/bar',
'X-Delete-At': t})
- with set_http_connect():
+ with set_http_connect_contextmgr():
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 400)
self.assertEqual('Non-integer X-Delete-At', resp.body)
@@ -435,14 +465,14 @@ class BaseObjectControllerMixin(object):
req = swob.Request.blank('/v1/a/c/o', method='PUT', body='',
headers={'Content-Type': 'foo/bar',
'X-Delete-At': t})
- with set_http_connect():
+ with set_http_connect_contextmgr():
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 400)
self.assertEqual('X-Delete-At in past', resp.body)
def test_HEAD_simple(self):
req = swift.common.swob.Request.blank('/v1/a/c/o', method='HEAD')
- with set_http_connect(200):
+ with set_http_connect_contextmgr(200):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
self.assertIn('Accept-Ranges', resp.headers)
@@ -450,7 +480,7 @@ class BaseObjectControllerMixin(object):
def test_HEAD_x_newest(self):
req = swift.common.swob.Request.blank('/v1/a/c/o', method='HEAD',
headers={'X-Newest': 'true'})
- with set_http_connect(*([200] * self.replicas())):
+ with set_http_connect_contextmgr(*([200] * self.replicas())):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
@@ -465,8 +495,8 @@ class BaseObjectControllerMixin(object):
'X-Backend-Timestamp': t.internal,
'X-Timestamp': t.normal
} for t in timestamps]
- with set_http_connect(*([200] * self.replicas()),
- headers=backend_response_headers):
+ with set_http_connect_contextmgr(*([200] * self.replicas()),
+ headers=backend_response_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
self.assertEqual(resp.headers['x-timestamp'], newest_timestamp.normal)
@@ -483,8 +513,8 @@ class BaseObjectControllerMixin(object):
'X-Backend-Timestamp': t.internal,
'X-Timestamp': t.normal
} for t in timestamps]
- with set_http_connect(*([200] * self.replicas()),
- headers=backend_response_headers):
+ with set_http_connect_contextmgr(*([200] * self.replicas()),
+ headers=backend_response_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
self.assertEqual(resp.headers['x-backend-timestamp'],
@@ -514,9 +544,9 @@ class BaseObjectControllerMixin(object):
'headers': headers,
}
request_log.append(req)
- with set_http_connect(*responses,
- headers=backend_response_headers,
- give_connect=capture_requests):
+ with set_http_connect_contextmgr(*responses,
+ headers=backend_response_headers,
+ give_connect=capture_requests):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
for req in request_log:
@@ -532,7 +562,7 @@ class BaseObjectControllerMixin(object):
'X-Timestamp': next(ts).internal})
codes = [409] * self.obj_ring.replicas
ts_iter = itertools.repeat(next(ts).internal)
- with set_http_connect(*codes, timestamps=ts_iter):
+ with set_http_connect_contextmgr(*codes, timestamps=ts_iter):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 409)
@@ -582,12 +612,12 @@ class BaseObjectControllerMixin(object):
class TestReplicatedObjController(BaseObjectControllerMixin,
unittest.TestCase):
- controller_cls = obj.ReplicatedObjectController
+ controller_cls = ReplicatedObjectController
def test_PUT_simple(self):
req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT')
req.headers['content-length'] = '0'
- with set_http_connect(201, 201, 201):
+ with set_http_connect_contextmgr(201, 201, 201):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -602,7 +632,8 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
'X-Obj-Metadata-Footer': 'yes'
}
- with set_http_connect(*codes, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(*codes,
+ expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 503)
@@ -634,10 +665,12 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
'Some-Header': 'Four',
'Etag': '"%s"' % etag,
}
- with set_http_connect(*codes, expect_headers=expect_headers,
- give_send=capture_body,
- give_connect=capture_headers,
- headers=resp_headers):
+ with set_http_connect_contextmgr(
+ *codes,
+ expect_headers=expect_headers,
+ give_send=capture_body,
+ give_connect=capture_headers,
+ headers=resp_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -714,10 +747,12 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
# NB: ignored!
'Some-Header': 'Four',
}
- with set_http_connect(*codes, expect_headers=expect_headers,
- give_send=capture_body,
- give_connect=capture_headers,
- headers=resp_headers):
+ with set_http_connect_contextmgr(
+ *codes,
+ expect_headers=expect_headers,
+ give_send=capture_body,
+ give_connect=capture_headers,
+ headers=resp_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -788,8 +823,8 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
# we capture stdout since the debug log formatter prints the formatted
# message to stdout
stdout = BytesIO()
- with set_http_connect((100, Timeout()), 503, 503), \
- mock.patch('sys.stdout', stdout):
+ with set_http_connect_contextmgr(
+ (100, Timeout()), 503, 503), mock.patch('sys.stdout', stdout):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 503)
for line in stdout.getvalue().splitlines():
@@ -813,7 +848,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
codes = [FakeStatus((422, 200))
for _junk in range(self.replicas())]
- with set_http_connect(*codes):
+ with set_http_connect_contextmgr(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 422)
@@ -821,7 +856,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT')
req.headers['if-none-match'] = '*'
req.headers['content-length'] = '0'
- with set_http_connect(201, 201, 201):
+ with set_http_connect_contextmgr(201, 201, 201):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -829,7 +864,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT')
req.headers['if-none-match'] = '*'
req.headers['content-length'] = '0'
- with set_http_connect(201, 412, 201):
+ with set_http_connect_contextmgr(201, 412, 201):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 412)
@@ -837,7 +872,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
req = swift.common.swob.Request.blank('/v1/a/c/o', method='PUT')
req.headers['if-none-match'] = 'somethingelse'
req.headers['content-length'] = '0'
- with set_http_connect():
+ with set_http_connect_contextmgr():
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 400)
@@ -849,7 +884,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
self.app._error_limiting = {}
req = swob.Request.blank('/v1/a/c/o.jpg', method='PUT',
body='test body')
- with set_http_connect(*statuses):
+ with set_http_connect_contextmgr(*statuses):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, expected)
@@ -904,7 +939,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
method='PUT',
body='life is utf-gr8')
self.app.logger.clear()
- with set_http_connect(*statuses):
+ with set_http_connect_contextmgr(*statuses):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, expected)
@@ -920,7 +955,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
method='PUT',
body='life is utf-gr8')
self.app.logger.clear()
- with set_http_connect(*statuses):
+ with set_http_connect_contextmgr(*statuses):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -948,7 +983,8 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
method='PUT',
body='life is utf-gr8')
self.app.logger.clear()
- with set_http_connect(201, 201, 201, give_send=capture_send):
+ with set_http_connect_contextmgr(201, 201, 201,
+ give_send=capture_send):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -967,7 +1003,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
method='PUT',
body='life is utf-gr8')
self.app.logger.clear()
- with set_http_connect(*statuses):
+ with set_http_connect_contextmgr(*statuses):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -1000,7 +1036,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
method='DELETE',
body='life is utf-gr8')
self.app.logger.clear()
- with set_http_connect(*statuses):
+ with set_http_connect_contextmgr(*statuses):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -1050,7 +1086,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
req.environ['wsgi.input'] = FakeReader()
req.headers['content-length'] = '6'
- with set_http_connect(201, 201, 201):
+ with set_http_connect_contextmgr(201, 201, 201):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 499)
@@ -1065,7 +1101,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
req.environ['wsgi.input'] = FakeReader()
req.headers['content-length'] = '6'
- with set_http_connect(201, 201, 201):
+ with set_http_connect_contextmgr(201, 201, 201):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 408)
@@ -1085,7 +1121,8 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
req.environ['wsgi.input'] = FakeReader()
req.headers['content-length'] = '6'
- with set_http_connect(201, 201, 201, give_expect=capture_expect):
+ with set_http_connect_contextmgr(201, 201, 201,
+ give_expect=capture_expect):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 499)
@@ -1103,29 +1140,32 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
req.environ['wsgi.input'] = FakeReader()
req.headers['content-length'] = '6'
- with set_http_connect(201, 201, 201):
+ with set_http_connect_contextmgr(201, 201, 201):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 500)
def test_GET_simple(self):
req = swift.common.swob.Request.blank('/v1/a/c/o')
- with set_http_connect(200):
+ with set_http_connect_contextmgr(200):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
self.assertIn('Accept-Ranges', resp.headers)
def test_GET_transfer_encoding_chunked(self):
req = swift.common.swob.Request.blank('/v1/a/c/o')
- with set_http_connect(200, headers={'transfer-encoding': 'chunked'}):
+ with set_http_connect_contextmgr(
+ 200,
+ headers={'transfer-encoding': 'chunked'}):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
self.assertEqual(resp.headers['Transfer-Encoding'], 'chunked')
def _test_removes_swift_bytes(self, method):
req = swift.common.swob.Request.blank('/v1/a/c/o', method=method)
- with set_http_connect(
- 200, headers={'content-type': 'image/jpeg; swift_bytes=99'}):
+ with set_http_connect_contextmgr(
+ 200,
+ headers={'content-type': 'image/jpeg; swift_bytes=99'}):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
self.assertEqual(resp.headers['Content-Type'], 'image/jpeg')
@@ -1140,7 +1180,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
req = swift.common.swob.Request.blank('/v1/a/c/o')
self.app.logger.txn_id = req.environ['swift.trans_id'] = 'my-txn-id'
stdout = BytesIO()
- with set_http_connect(503, 200), \
+ with set_http_connect_contextmgr(503, 200), \
mock.patch('sys.stdout', stdout):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
@@ -1151,7 +1191,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
def test_GET_handoff(self):
req = swift.common.swob.Request.blank('/v1/a/c/o')
codes = [503] * self.obj_ring.replicas + [200]
- with set_http_connect(*codes):
+ with set_http_connect_contextmgr(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
@@ -1159,7 +1199,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
req = swift.common.swob.Request.blank('/v1/a/c/o')
codes = [404] * (self.obj_ring.replicas +
self.obj_ring.max_more_nodes)
- with set_http_connect(*codes):
+ with set_http_connect_contextmgr(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 404)
@@ -1175,7 +1215,8 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
if method == 'PUT':
put_headers.append(headers)
codes = [201] * self.obj_ring.replicas
- with set_http_connect(*codes, give_connect=capture_headers):
+ with set_http_connect_contextmgr(*codes,
+ give_connect=capture_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
for given_headers in put_headers:
@@ -1197,7 +1238,8 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
put_headers.append(headers)
codes = [201] * self.obj_ring.replicas
t = time.time()
- with set_http_connect(*codes, give_connect=capture_headers):
+ with set_http_connect_contextmgr(*codes,
+ give_connect=capture_headers):
with mock.patch('time.time', lambda: t):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -1220,7 +1262,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
'Content-Length': 0,
'X-Timestamp': put_timestamp})
codes = [201] * self.obj_ring.replicas
- with set_http_connect(*codes):
+ with set_http_connect_contextmgr(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -1235,7 +1277,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
'X-Timestamp': put_timestamp})
ts_iter = itertools.repeat(put_timestamp)
codes = [409] * self.obj_ring.replicas
- with set_http_connect(*codes, timestamps=ts_iter):
+ with set_http_connect_contextmgr(*codes, timestamps=ts_iter):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 202)
@@ -1250,7 +1292,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
'X-Timestamp': next(ts).internal})
ts_iter = itertools.repeat(next(ts).internal)
codes = [409] * self.obj_ring.replicas
- with set_http_connect(*codes, timestamps=ts_iter):
+ with set_http_connect_contextmgr(*codes, timestamps=ts_iter):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 202)
@@ -1265,7 +1307,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
'X-Timestamp': next(ts).internal})
ts_iter = itertools.repeat(orig_timestamp)
codes = [201] * self.obj_ring.replicas
- with set_http_connect(*codes, timestamps=ts_iter):
+ with set_http_connect_contextmgr(*codes, timestamps=ts_iter):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -1277,7 +1319,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
'X-Timestamp': next(ts).internal})
ts_iter = iter([next(ts).internal, None, None])
codes = [409] + [201] * (self.obj_ring.replicas - 1)
- with set_http_connect(*codes, timestamps=ts_iter):
+ with set_http_connect_contextmgr(*codes, timestamps=ts_iter):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 202)
@@ -1289,7 +1331,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
'X-Timestamp': next(ts).internal})
ts_iter = iter([None, None, None])
codes = [409] * self.obj_ring.replicas
- with set_http_connect(*codes, timestamps=ts_iter):
+ with set_http_connect_contextmgr(*codes, timestamps=ts_iter):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 202)
@@ -1301,7 +1343,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
'X-Timestamp': next(ts).internal})
ts_iter = iter([next(ts).internal, None, None])
codes = [409] + [(201, 'notused')] * (self.obj_ring.replicas - 1)
- with set_http_connect(*codes, timestamps=ts_iter):
+ with set_http_connect_contextmgr(*codes, timestamps=ts_iter):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 202)
@@ -1314,7 +1356,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
'X-Timestamp': next(ts).internal})
ts_iter = iter([next(ts).internal, None, None])
codes = [409] + [(412, 'notused')] * (self.obj_ring.replicas - 1)
- with set_http_connect(*codes, timestamps=ts_iter):
+ with set_http_connect_contextmgr(*codes, timestamps=ts_iter):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 412)
@@ -1334,7 +1376,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
codes = [409] * self.obj_ring.replicas
ts_iter = iter(put_ts)
- with set_http_connect(*codes, timestamps=ts_iter):
+ with set_http_connect_contextmgr(*codes, timestamps=ts_iter):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 202)
@@ -1356,7 +1398,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
ts_iter = iter(put_ts)
codes = put_resp
- with set_http_connect(*codes, timestamps=ts_iter):
+ with set_http_connect_contextmgr(*codes, timestamps=ts_iter):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 202)
@@ -1370,7 +1412,7 @@ class TestReplicatedObjController(BaseObjectControllerMixin,
{'replicas': 1}, {'replicas': 5}, {'replicas': 8}, {'replicas': 15}])
class TestReplicatedObjControllerVariousReplicas(BaseObjectControllerMixin,
unittest.TestCase):
- controller_cls = obj.ReplicatedObjectController
+ controller_cls = ReplicatedObjectController
class StubResponse(object):
@@ -1456,7 +1498,7 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'storage_policy': '0',
}
- controller_cls = obj.ECObjectController
+ controller_cls = ECObjectController
def _add_frag_index(self, index, headers):
# helper method to add a frag index header to an existing header dict
@@ -1508,7 +1550,7 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
def test_GET_simple(self):
req = swift.common.swob.Request.blank('/v1/a/c/o')
get_resp = [200] * self.policy.ec_ndata
- with set_http_connect(*get_resp):
+ with set_http_connect_contextmgr(*get_resp):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
self.assertIn('Accept-Ranges', resp.headers)
@@ -1526,7 +1568,7 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
resp_headers = {'Etag': 'frag_etag',
'X-Object-Sysmeta-Ec-Etag': 'data_etag',
'X-Object-Sysmeta-Alternate-Etag': 'alt_etag'}
- with set_http_connect(*get_resp, headers=resp_headers):
+ with set_http_connect_contextmgr(*get_resp, headers=resp_headers):
resp = req.get_response(self.app)
self.assertEqual('data_etag', resp.headers['Etag'])
return resp
@@ -1577,7 +1619,7 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
resp_headers = {'Etag': 'frag_etag',
'X-Object-Sysmeta-Ec-Etag': 'data_etag',
'X-Object-Sysmeta-Alternate-Etag': 'alt_etag'}
- with set_http_connect(*get_resp, headers=resp_headers):
+ with set_http_connect_contextmgr(*get_resp, headers=resp_headers):
resp = req.get_response(self.app)
self.assertEqual('data_etag', resp.headers['Etag'])
return resp
@@ -1619,14 +1661,14 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
req = swift.common.swob.Request.blank('/v1/a/c/o',
headers={'X-Newest': 'true'})
codes = [200] * self.policy.ec_ndata
- with set_http_connect(*codes):
+ with set_http_connect_contextmgr(*codes):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
def test_GET_error(self):
req = swift.common.swob.Request.blank('/v1/a/c/o')
get_resp = [503] + [200] * self.policy.ec_ndata
- with set_http_connect(*get_resp):
+ with set_http_connect_contextmgr(*get_resp):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
@@ -1659,8 +1701,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
responses = [(200, ''.join(node_fragments[i]), headers)
for i in range(POLICIES.default.ec_ndata)]
status_codes, body_iter, headers = zip(*responses)
- with set_http_connect(*status_codes, body_iter=body_iter,
- headers=headers):
+ with set_http_connect_contextmgr(*status_codes, body_iter=body_iter,
+ headers=headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
self.assertEqual(len(real_body), len(resp.body))
@@ -1674,7 +1716,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
- with set_http_connect(*codes, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(*codes,
+ expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -1697,8 +1740,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
headers = {'Etag': 'bad etag'}
req = swift.common.swob.Request.blank(
'/v1/a/c/o', method='PUT', headers=headers, body=test_body)
- with set_http_connect(*codes, expect_headers=expect_headers,
- give_expect=capture_expect):
+ with set_http_connect_contextmgr(*codes, expect_headers=expect_headers,
+ give_expect=capture_expect):
resp = req.get_response(self.app)
self.assertEqual(422, resp.status_int)
self.assertEqual(self.replicas(), len(conns))
@@ -1710,7 +1753,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
env = {'swift.callback.update_footers': footers_callback}
req = swift.common.swob.Request.blank(
'/v1/a/c/o', method='PUT', environ=env, body=test_body)
- with set_http_connect(*codes, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(*codes,
+ expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(422, resp.status_int)
@@ -1724,7 +1768,7 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
- with set_http_connect(*codes, expect_headers=expect_headers), \
+ with set_http_connect_contextmgr(*codes, expect_headers=expect_headers), \
mock.patch('sys.stdout', stdout):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 503)
@@ -1741,7 +1785,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
- with set_http_connect(*codes, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(*codes,
+ expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -1753,7 +1798,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
- with set_http_connect(*codes, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(*codes,
+ expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 503)
@@ -1767,7 +1813,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
- with set_http_connect(*codes, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(*codes,
+ expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -1779,7 +1826,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
- with set_http_connect(*codes, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(*codes,
+ expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 503)
@@ -1794,7 +1842,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
- with set_http_connect(*codes, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(*codes,
+ expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -1808,7 +1857,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
- with set_http_connect(*codes, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(*codes,
+ expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 503)
@@ -1821,7 +1871,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
- with set_http_connect(*codes, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(*codes,
+ expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -1834,7 +1885,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
- with set_http_connect(*codes, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(*codes,
+ expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -1853,7 +1905,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
- with set_http_connect(*codes, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(*codes,
+ expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 499)
@@ -1873,7 +1926,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
- with set_http_connect(*codes, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(*codes,
+ expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 408)
@@ -1893,7 +1947,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
- with set_http_connect(*codes, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(*codes,
+ expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 499)
@@ -1913,7 +1968,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
- with set_http_connect(*codes, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(*codes,
+ expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 500)
@@ -1956,10 +2012,12 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
put_requests[conn_id]['x-timestamp'] = headers[
'X-Timestamp']
- with set_http_connect(*codes, expect_headers=expect_headers,
- give_send=capture_body,
- give_connect=capture_headers,
- headers=resp_headers):
+ with set_http_connect_contextmgr(
+ *codes,
+ expect_headers=expect_headers,
+ give_send=capture_body,
+ give_connect=capture_headers,
+ headers=resp_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -2089,10 +2147,12 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
req = swift.common.swob.Request.blank(
'/v1/a/c/o', method='PUT', environ=env, body=test_body)
- with set_http_connect(*codes, expect_headers=expect_headers,
- give_send=capture_body,
- give_connect=capture_headers,
- headers=resp_headers):
+ with set_http_connect_contextmgr(
+ *codes,
+ expect_headers=expect_headers,
+ give_send=capture_body,
+ give_connect=capture_headers,
+ headers=resp_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -2200,7 +2260,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
# whoops, stupid random
responses = responses[1:] + [responses[0]]
codes, expect_headers = zip(*responses)
- with set_http_connect(*codes, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(*codes,
+ expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -2320,7 +2381,7 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'X-Timestamp': obj_stub['timestamp'].normal,
'X-Backend-Data-Timestamp': obj_stub['timestamp'].internal,
'X-Backend-Fragments':
- server._make_backend_fragments_header(ts2frags)
+ object_server._make_backend_fragments_header(ts2frags)
}
if durable_timestamp:
headers['X-Backend-Durable-Timestamp'] = durable_timestamp
@@ -3536,8 +3597,10 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
# sanity check responses1
responses = responses1[:self.policy.ec_ndata]
status_codes, body_iter, headers = zip(*responses)
- with set_http_connect(*status_codes, body_iter=body_iter,
- headers=headers):
+ with set_http_connect_contextmgr(
+ *status_codes,
+ body_iter=body_iter,
+ headers=headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
self.assertEqual(md5(resp.body).hexdigest(), etag1)
@@ -3545,8 +3608,10 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
# sanity check responses2
responses = responses2[:self.policy.ec_ndata]
status_codes, body_iter, headers = zip(*responses)
- with set_http_connect(*status_codes, body_iter=body_iter,
- headers=headers):
+ with set_http_connect_contextmgr(
+ *status_codes,
+ body_iter=body_iter,
+ headers=headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
self.assertEqual(md5(resp.body).hexdigest(), etag2)
@@ -3557,8 +3622,10 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
mixed_responses[mix_index] = responses2[mix_index]
status_codes, body_iter, headers = zip(*mixed_responses)
- with set_http_connect(*status_codes, body_iter=body_iter,
- headers=headers):
+ with set_http_connect_contextmgr(
+ *status_codes,
+ body_iter=body_iter,
+ headers=headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
try:
@@ -3591,8 +3658,10 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
status_codes, body_iter, headers = zip(*responses + [
(404, '', {}) for i in range(
self.policy.object_ring.max_more_nodes)])
- with set_http_connect(*status_codes, body_iter=body_iter,
- headers=headers):
+ with set_http_connect_contextmgr(
+ *status_codes,
+ body_iter=body_iter,
+ headers=headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
# do this inside the fake http context manager, it'll try to
@@ -3624,8 +3693,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
status_codes, body_iter, headers = zip(
*responses[:self.policy.ec_ndata + 1])
- with set_http_connect(*status_codes, body_iter=body_iter,
- headers=headers):
+ with set_http_connect_contextmgr(*status_codes, body_iter=body_iter,
+ headers=headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
self.assertTrue(md5(resp.body).hexdigest(), etag)
@@ -3641,8 +3710,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
responses = [(200, '', headers)]
status_codes, body_iter, headers = zip(*responses)
req = swift.common.swob.Request.blank('/v1/a/c/o', method='HEAD')
- with set_http_connect(*status_codes, body_iter=body_iter,
- headers=headers):
+ with set_http_connect_contextmgr(*status_codes, body_iter=body_iter,
+ headers=headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 200)
self.assertEqual(resp.body, '')
@@ -3654,8 +3723,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
responses = [(404, '', {})] * self.replicas() * 2
status_codes, body_iter, headers = zip(*responses)
req = swift.common.swob.Request.blank('/v1/a/c/o', method='HEAD')
- with set_http_connect(*status_codes, body_iter=body_iter,
- headers=headers):
+ with set_http_connect_contextmgr(*status_codes, body_iter=body_iter,
+ headers=headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 404)
# 404 shows actual response body size (i.e. 0 for HEAD)
@@ -3684,7 +3753,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
- with set_http_connect(*codes, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(*codes,
+ expect_headers=expect_headers):
start = time.time()
resp = req.get_response(self.app)
response_time = time.time() - start
@@ -3703,7 +3773,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
- with set_http_connect(*codes, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(*codes,
+ expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 201)
@@ -3719,7 +3790,8 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
- with set_http_connect(*codes, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(*codes,
+ expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 503)
@@ -3768,8 +3840,11 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
'X-Obj-Metadata-Footer': 'yes',
'X-Obj-Multiphase-Commit': 'yes'
}
- with set_http_connect(*status_codes, body_iter=body_iter,
- headers=headers, expect_headers=expect_headers):
+ with set_http_connect_contextmgr(
+ *status_codes,
+ body_iter=body_iter,
+ headers=headers,
+ expect_headers=expect_headers):
resp = req.get_response(self.app)
self.assertEqual(resp.status_int, 416)
self.assertEqual(resp.content_length, len(range_not_satisfiable_body))
@@ -3778,5 +3853,2135 @@ class TestECObjController(BaseObjectControllerMixin, unittest.TestCase):
self.assertEqual(resp.headers['Accept-Ranges'], 'bytes')
+@patch_policies([StoragePolicy(0, 'zero', True,
+ object_ring=FakeRing(base_port=3000))])
+class TestObjectController(unittest.TestCase):
+
+ def setUp(self):
+ self.app = proxy_server.Application(
+ None, FakeMemcache(),
+ logger=debug_logger('proxy-ut'),
+ account_ring=FakeRing(),
+ container_ring=FakeRing())
+ # clear proxy logger result for each test
+ _test_servers[0].logger._clear()
+
+ def tearDown(self):
+ self.app.account_ring.set_replicas(3)
+ self.app.container_ring.set_replicas(3)
+ for policy in POLICIES:
+ policy.object_ring = FakeRing(base_port=3000)
+
+ def assert_status_map(self, method, statuses, expected, raise_exc=False):
+ with save_globals():
+ kwargs = {}
+ if raise_exc:
+ kwargs['raise_exc'] = raise_exc
+
+ set_http_connect(*statuses, **kwargs)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a/c/o',
+ headers={'Content-Length': '0',
+ 'Content-Type': 'text/plain'})
+ self.app.update_request(req)
+ try:
+ res = method(req)
+ except HTTPException as res:
+ pass
+ self.assertEqual(res.status_int, expected)
+
+ # repeat test
+ set_http_connect(*statuses, **kwargs)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a/c/o',
+ headers={'Content-Length': '0',
+ 'Content-Type': 'text/plain'})
+ self.app.update_request(req)
+ try:
+ res = method(req)
+ except HTTPException as res:
+ pass
+ self.assertEqual(res.status_int, expected)
+
+ def _sleep_enough(self, condition):
+ for sleeptime in (0.1, 1.0):
+ sleep(sleeptime)
+ if condition():
+ break
+
+ def test_PUT_expect_header_zero_content_length(self):
+ test_errors = []
+
+ def test_connect(ipaddr, port, device, partition, method, path,
+ headers=None, query_string=None):
+ if path == '/a/c/o.jpg':
+ if 'expect' in headers or 'Expect' in headers:
+ test_errors.append('Expect was in headers for object '
+ 'server!')
+
+ with save_globals():
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ # The (201, Exception('test')) tuples in there have the effect of
+ # changing the status of the initial expect response. The default
+ # expect response from FakeConn for 201 is 100.
+ # But the object server won't send a 100 continue line if the
+ # client doesn't send a expect 100 header (as is the case with
+ # zero byte PUTs as validated by this test), nevertheless the
+ # object controller calls getexpect without prejudice. In this
+ # case the status from the response shows up early in getexpect
+ # instead of having to wait until getresponse. The Exception is
+ # in there to ensure that the object controller also *uses* the
+ # result of getexpect instead of calling getresponse in which case
+ # our FakeConn will blow up.
+ success_codes = [(201, Exception('test'))] * 3
+ set_http_connect(200, 200, *success_codes,
+ give_connect=test_connect)
+ req = Request.blank('/v1/a/c/o.jpg', {})
+ req.content_length = 0
+ self.app.update_request(req)
+ self.app.memcache.store = {}
+ res = controller.PUT(req)
+ self.assertEqual(test_errors, [])
+ self.assertTrue(res.status.startswith('201 '), res.status)
+
+ def test_PUT_expect_header_nonzero_content_length(self):
+ test_errors = []
+
+ def test_connect(ipaddr, port, device, partition, method, path,
+ headers=None, query_string=None):
+ if path == '/a/c/o.jpg':
+ if 'Expect' not in headers:
+ test_errors.append('Expect was not in headers for '
+ 'non-zero byte PUT!')
+
+ with save_globals():
+ controller = ReplicatedObjectController(
+ self.app, 'a', 'c', 'o.jpg')
+ # the (100, 201) tuples in there are just being extra explicit
+ # about the FakeConn returning the 100 Continue status when the
+ # object controller calls getexpect. Which is FakeConn's default
+ # for 201 if no expect_status is specified.
+ success_codes = [(100, 201)] * 3
+ set_http_connect(200, 200, *success_codes,
+ give_connect=test_connect)
+ req = Request.blank('/v1/a/c/o.jpg', {})
+ req.content_length = 1
+ req.body = 'a'
+ self.app.update_request(req)
+ self.app.memcache.store = {}
+ res = controller.PUT(req)
+ self.assertEqual(test_errors, [])
+ self.assertTrue(res.status.startswith('201 '))
+
+ def test_PUT_respects_write_affinity(self):
+ written_to = []
+
+ def test_connect(ipaddr, port, device, partition, method, path,
+ headers=None, query_string=None):
+ if path == '/a/c/o.jpg':
+ written_to.append((ipaddr, port, device))
+
+ with save_globals():
+ def is_r0(node):
+ return node['region'] == 0
+
+ object_ring = self.app.get_object_ring(None)
+ object_ring.max_more_nodes = 100
+ self.app.write_affinity_is_local_fn = is_r0
+ self.app.write_affinity_node_count = lambda r: 3
+
+ controller = \
+ ReplicatedObjectController(
+ self.app, 'a', 'c', 'o.jpg')
+ set_http_connect(200, 200, 201, 201, 201,
+ give_connect=test_connect)
+ req = Request.blank('/v1/a/c/o.jpg', {})
+ req.content_length = 1
+ req.body = 'a'
+ self.app.memcache.store = {}
+ res = controller.PUT(req)
+ self.assertTrue(res.status.startswith('201 '))
+
+ self.assertEqual(3, len(written_to))
+ for ip, port, device in written_to:
+ # this is kind of a hokey test, but in FakeRing, the port is even
+ # when the region is 0, and odd when the region is 1, so this test
+ # asserts that we only wrote to nodes in region 0.
+ self.assertEqual(0, port % 2)
+
+ def test_PUT_respects_write_affinity_with_507s(self):
+ written_to = []
+
+ def test_connect(ipaddr, port, device, partition, method, path,
+ headers=None, query_string=None):
+ if path == '/a/c/o.jpg':
+ written_to.append((ipaddr, port, device))
+
+ with save_globals():
+ def is_r0(node):
+ return node['region'] == 0
+
+ object_ring = self.app.get_object_ring(None)
+ object_ring.max_more_nodes = 100
+ self.app.write_affinity_is_local_fn = is_r0
+ self.app.write_affinity_node_count = lambda r: 3
+
+ controller = \
+ ReplicatedObjectController(
+ self.app, 'a', 'c', 'o.jpg')
+ self.app.error_limit(
+ object_ring.get_part_nodes(1)[0], 'test')
+ set_http_connect(200, 200, # account, container
+ 201, 201, 201, # 3 working backends
+ give_connect=test_connect)
+ req = Request.blank('/v1/a/c/o.jpg', {})
+ req.content_length = 1
+ req.body = 'a'
+ self.app.memcache.store = {}
+ res = controller.PUT(req)
+ self.assertTrue(res.status.startswith('201 '))
+
+ self.assertEqual(3, len(written_to))
+ # this is kind of a hokey test, but in FakeRing, the port is even when
+ # the region is 0, and odd when the region is 1, so this test asserts
+ # that we wrote to 2 nodes in region 0, then went to 1 non-r0 node.
+ self.assertEqual(0, written_to[0][1] % 2) # it's (ip, port, device)
+ self.assertEqual(0, written_to[1][1] % 2)
+ self.assertNotEqual(0, written_to[2][1] % 2)
+
+ def test_PUT_auto_content_type(self):
+ with save_globals():
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+
+ def test_content_type(filename, expected):
+ # The three responses here are for account_info() (HEAD to
+ # account server), container_info() (HEAD to container server)
+ # and three calls to _connect_put_node() (PUT to three object
+ # servers)
+ set_http_connect(201, 201, 201, 201, 201,
+ give_content_type=lambda content_type:
+ self.assertEqual(content_type,
+ next(expected)))
+ # We need into include a transfer-encoding to get past
+ # constraints.check_object_creation()
+ req = Request.blank('/v1/a/c/%s' % filename, {},
+ headers={'transfer-encoding': 'chunked'})
+ self.app.update_request(req)
+ self.app.memcache.store = {}
+ res = controller.PUT(req)
+ # If we don't check the response here we could miss problems
+ # in PUT()
+ self.assertEqual(res.status_int, 201)
+
+ test_content_type('test.jpg', iter(['', '', 'image/jpeg',
+ 'image/jpeg', 'image/jpeg']))
+ test_content_type('test.html', iter(['', '', 'text/html',
+ 'text/html', 'text/html']))
+ test_content_type('test.css', iter(['', '', 'text/css',
+ 'text/css', 'text/css']))
+
+ def test_custom_mime_types_files(self):
+ swift_dir = mkdtemp()
+ try:
+ with open(os.path.join(swift_dir, 'mime.types'), 'w') as fp:
+ fp.write('foo/bar foo\n')
+ proxy_server.Application({'swift_dir': swift_dir},
+ FakeMemcache(), FakeLogger(),
+ FakeRing(), FakeRing())
+ self.assertEqual(proxy_server.mimetypes.guess_type('blah.foo')[0],
+ 'foo/bar')
+ self.assertEqual(proxy_server.mimetypes.guess_type('blah.jpg')[0],
+ 'image/jpeg')
+ finally:
+ rmtree(swift_dir, ignore_errors=True)
+
+ def test_PUT(self):
+ with save_globals():
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+
+ def test_status_map(statuses, expected):
+ set_http_connect(*statuses)
+ req = Request.blank('/v1/a/c/o.jpg', {})
+ req.content_length = 0
+ self.app.update_request(req)
+ self.app.memcache.store = {}
+ res = controller.PUT(req)
+ expected = str(expected)
+ self.assertEqual(res.status[:len(expected)], expected)
+ test_status_map((200, 200, 201, 201, 201), 201)
+ test_status_map((200, 200, 201, 201, 500), 201)
+ test_status_map((200, 200, 204, 404, 404), 404)
+ test_status_map((200, 200, 204, 500, 404), 503)
+ test_status_map((200, 200, 202, 202, 204), 204)
+
+ def test_PUT_connect_exceptions(self):
+ with save_globals():
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+
+ def test_status_map(statuses, expected):
+ set_http_connect(*statuses)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a/c/o.jpg', {})
+ req.content_length = 0
+ self.app.update_request(req)
+ try:
+ res = controller.PUT(req)
+ except HTTPException as res:
+ pass
+ expected = str(expected)
+ self.assertEqual(res.status[:len(expected)], expected)
+ test_status_map((200, 200, 201, 201, -1), 201) # connect exc
+ # connect errors
+ test_status_map((200, 200, Timeout(), 201, 201, ), 201)
+ test_status_map((200, 200, 201, 201, Exception()), 201)
+ # expect errors
+ test_status_map((200, 200, (Timeout(), None), 201, 201), 201)
+ test_status_map((200, 200, (Exception(), None), 201, 201), 201)
+ # response errors
+ test_status_map((200, 200, (100, Timeout()), 201, 201), 201)
+ test_status_map((200, 200, (100, Exception()), 201, 201), 201)
+ test_status_map((200, 200, 507, 201, 201), 201) # error limited
+ test_status_map((200, 200, -1, 201, -1), 503)
+ test_status_map((200, 200, 503, -1, 503), 503)
+
+ def test_PUT_send_exceptions(self):
+ with save_globals():
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+
+ def test_status_map(statuses, expected):
+ self.app.memcache.store = {}
+ set_http_connect(*statuses)
+ req = Request.blank('/v1/a/c/o.jpg',
+ environ={'REQUEST_METHOD': 'PUT'},
+ body='some data')
+ self.app.update_request(req)
+ try:
+ res = controller.PUT(req)
+ except HTTPException as res:
+ pass
+ expected = str(expected)
+ self.assertEqual(res.status[:len(expected)], expected)
+ test_status_map((200, 200, 201, -1, 201), 201)
+ test_status_map((200, 200, 201, -1, -1), 503)
+ test_status_map((200, 200, 503, 503, -1), 503)
+
+ def test_PUT_max_size(self):
+ with save_globals():
+ set_http_connect(201, 201, 201)
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ req = Request.blank('/v1/a/c/o', {}, headers={
+ 'Content-Length': str(constraints.MAX_FILE_SIZE + 1),
+ 'Content-Type': 'foo/bar'})
+ self.app.update_request(req)
+ res = controller.PUT(req)
+ self.assertEqual(res.status_int, 413)
+
+ def test_PUT_bad_content_type(self):
+ with save_globals():
+ set_http_connect(201, 201, 201)
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ req = Request.blank('/v1/a/c/o', {}, headers={
+ 'Content-Length': 0, 'Content-Type': 'foo/bar;swift_hey=45'})
+ self.app.update_request(req)
+ res = controller.PUT(req)
+ self.assertEqual(res.status_int, 400)
+
+ def test_PUT_getresponse_exceptions(self):
+
+ with save_globals():
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+
+ def test_status_map(statuses, expected):
+ self.app.memcache.store = {}
+ set_http_connect(*statuses)
+ req = Request.blank('/v1/a/c/o.jpg', {})
+ req.content_length = 0
+ self.app.update_request(req)
+ try:
+ res = controller.PUT(req)
+ except HTTPException as res:
+ pass
+ expected = str(expected)
+ self.assertEqual(res.status[:len(str(expected))],
+ str(expected))
+ test_status_map((200, 200, 201, 201, -1), 201)
+ test_status_map((200, 200, 201, -1, -1), 503)
+ test_status_map((200, 200, 503, 503, -1), 503)
+
+ def test_POST(self):
+ with save_globals():
+ self.app.object_post_as_copy = False
+
+ def test_status_map(statuses, expected):
+ set_http_connect(*statuses)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a/c/o', {}, method='POST',
+ headers={'Content-Type': 'foo/bar'})
+ self.app.update_request(req)
+ res = req.get_response(self.app)
+ expected = str(expected)
+ self.assertEqual(res.status[:len(expected)], expected)
+ test_status_map((200, 200, 202, 202, 202), 202)
+ test_status_map((200, 200, 202, 202, 500), 202)
+ test_status_map((200, 200, 202, 500, 500), 503)
+ test_status_map((200, 200, 202, 404, 500), 503)
+ test_status_map((200, 200, 202, 404, 404), 404)
+ test_status_map((200, 200, 404, 500, 500), 503)
+ test_status_map((200, 200, 404, 404, 404), 404)
+
+ @patch_policies([
+ StoragePolicy(0, 'zero', is_default=True, object_ring=FakeRing()),
+ StoragePolicy(1, 'one', object_ring=FakeRing()),
+ ])
+ def test_POST_backend_headers(self):
+ # reset the router post patch_policies
+ self.app.obj_controller_router = proxy_server.ObjectControllerRouter()
+ self.app.object_post_as_copy = False
+ self.app.sort_nodes = lambda nodes: nodes
+ backend_requests = []
+
+ def capture_requests(ip, port, method, path, headers, *args,
+ **kwargs):
+ backend_requests.append((method, path, headers))
+
+ req = Request.blank('/v1/a/c/o', {}, method='POST',
+ headers={'X-Object-Meta-Color': 'Blue',
+ 'Content-Type': 'text/plain'})
+
+ # we want the container_info response to says a policy index of 1
+ resp_headers = {'X-Backend-Storage-Policy-Index': 1}
+ with mocked_http_conn(
+ 200, 200, 202, 202, 202,
+ headers=resp_headers, give_connect=capture_requests
+ ) as fake_conn:
+ resp = req.get_response(self.app)
+ self.assertRaises(StopIteration, fake_conn.code_iter.next)
+
+ self.assertEqual(resp.status_int, 202)
+ self.assertEqual(len(backend_requests), 5)
+
+ def check_request(req, method, path, headers=None):
+ req_method, req_path, req_headers = req
+ self.assertEqual(method, req_method)
+ # caller can ignore leading path parts
+ self.assertTrue(req_path.endswith(path),
+ 'expected path to end with %s, it was %s' % (
+ path, req_path))
+ headers = headers or {}
+ # caller can ignore some headers
+ for k, v in headers.items():
+ self.assertEqual(req_headers[k], v)
+ account_request = backend_requests.pop(0)
+ check_request(account_request, method='HEAD', path='/sda/0/a')
+ container_request = backend_requests.pop(0)
+ check_request(container_request, method='HEAD', path='/sda/0/a/c')
+ # make sure backend requests included expected container headers
+ container_headers = {}
+ for request in backend_requests:
+ req_headers = request[2]
+ device = req_headers['x-container-device']
+ host = req_headers['x-container-host']
+ container_headers[device] = host
+ expectations = {
+ 'method': 'POST',
+ 'path': '/0/a/c/o',
+ 'headers': {
+ 'X-Container-Partition': '0',
+ 'Connection': 'close',
+ 'User-Agent': 'proxy-server %s' % os.getpid(),
+ 'Host': 'localhost:80',
+ 'Referer': 'POST http://localhost/v1/a/c/o',
+ 'X-Object-Meta-Color': 'Blue',
+ 'X-Backend-Storage-Policy-Index': '1'
+ },
+ }
+ check_request(request, **expectations)
+
+ expected = {}
+ for i, device in enumerate(['sda', 'sdb', 'sdc']):
+ expected[device] = '10.0.0.%d:100%d' % (i, i)
+ self.assertEqual(container_headers, expected)
+
+ # and again with policy override
+ self.app.memcache.store = {}
+ backend_requests = []
+ req = Request.blank('/v1/a/c/o', {}, method='POST',
+ headers={'X-Object-Meta-Color': 'Blue',
+ 'Content-Type': 'text/plain',
+ 'X-Backend-Storage-Policy-Index': 0})
+ with mocked_http_conn(
+ 200, 200, 202, 202, 202,
+ headers=resp_headers, give_connect=capture_requests
+ ) as fake_conn:
+ resp = req.get_response(self.app)
+ self.assertRaises(StopIteration, fake_conn.code_iter.next)
+ self.assertEqual(resp.status_int, 202)
+ self.assertEqual(len(backend_requests), 5)
+ for request in backend_requests[2:]:
+ expectations = {
+ 'method': 'POST',
+ 'path': '/0/a/c/o', # ignore device bit
+ 'headers': {
+ 'X-Object-Meta-Color': 'Blue',
+ 'X-Backend-Storage-Policy-Index': '0',
+ }
+ }
+ check_request(request, **expectations)
+
+ def test_DELETE(self):
+ # import pdb; pdb.set_trace()
+ with save_globals():
+ def test_status_map(statuses, expected):
+ set_http_connect(*statuses)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'DELETE'})
+ self.app.update_request(req)
+ res = req.get_response(self.app)
+ self.assertEqual(res.status[:len(str(expected))],
+ str(expected))
+ test_status_map((200, 200, 204, 204, 204), 204)
+ test_status_map((200, 200, 204, 204, 500), 204)
+ test_status_map((200, 200, 204, 404, 404), 404)
+ test_status_map((200, 204, 500, 500, 404), 503)
+ test_status_map((200, 200, 404, 404, 404), 404)
+ test_status_map((200, 200, 400, 400, 400), 400)
+
+ def test_HEAD(self):
+ with save_globals():
+ def test_status_map(statuses, expected):
+ set_http_connect(*statuses)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'})
+ self.app.update_request(req)
+ res = req.get_response(self.app)
+ self.assertEqual(res.status[:len(str(expected))],
+ str(expected))
+ if expected < 400:
+ self.assertIn('x-works', res.headers)
+ self.assertEqual(res.headers['x-works'], 'yes')
+ self.assertIn('accept-ranges', res.headers)
+ self.assertEqual(res.headers['accept-ranges'], 'bytes')
+
+ test_status_map((200, 200, 200, 404, 404), 200)
+ test_status_map((200, 200, 200, 500, 404), 200)
+ test_status_map((200, 200, 304, 500, 404), 304)
+ test_status_map((200, 200, 404, 404, 404), 404)
+ test_status_map((200, 200, 404, 404, 500), 404)
+ test_status_map((200, 200, 500, 500, 500), 503)
+
+ def test_HEAD_newest(self):
+ with save_globals():
+ def test_status_map(statuses, expected, timestamps,
+ expected_timestamp):
+ set_http_connect(*statuses, timestamps=timestamps)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'},
+ headers={'x-newest': 'true'})
+ self.app.update_request(req)
+ res = req.get_response(self.app)
+ self.assertEqual(res.status[:len(str(expected))],
+ str(expected))
+ self.assertEqual(res.headers.get('last-modified'),
+ expected_timestamp)
+
+ # acct cont obj obj obj
+ test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
+ '2', '3'), '3')
+ test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
+ '3', '2'), '3')
+ test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
+ '3', '1'), '3')
+ test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '3',
+ '3', '1'), '3')
+ test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None,
+ None, None), None)
+ test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None,
+ None, '1'), '1')
+ test_status_map((200, 200, 404, 404, 200), 200, ('0', '0', None,
+ None, '1'), '1')
+
+ def test_GET_newest(self):
+ with save_globals():
+ def test_status_map(statuses, expected, timestamps,
+ expected_timestamp):
+ set_http_connect(*statuses, timestamps=timestamps)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'GET'},
+ headers={'x-newest': 'true'})
+ self.app.update_request(req)
+ res = req.get_response(self.app)
+ self.assertEqual(res.status[:len(str(expected))],
+ str(expected))
+ self.assertEqual(res.headers.get('last-modified'),
+ expected_timestamp)
+
+ test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
+ '2', '3'), '3')
+ test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
+ '3', '2'), '3')
+ test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
+ '3', '1'), '3')
+ test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '3',
+ '3', '1'), '3')
+ test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None,
+ None, None), None)
+ test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None,
+ None, '1'), '1')
+
+ with save_globals():
+ def test_status_map(statuses, expected, timestamps,
+ expected_timestamp):
+ set_http_connect(*statuses, timestamps=timestamps)
+ self.app.memcache.store = {}
+ req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'})
+ self.app.update_request(req)
+ res = req.get_response(self.app)
+ self.assertEqual(res.status[:len(str(expected))],
+ str(expected))
+ self.assertEqual(res.headers.get('last-modified'),
+ expected_timestamp)
+
+ test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
+ '2', '3'), '1')
+ test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
+ '3', '2'), '1')
+ test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '1',
+ '3', '1'), '1')
+ test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', '3',
+ '3', '1'), '3')
+ test_status_map((200, 200, 200, 200, 200), 200, ('0', '0', None,
+ '1', '2'), None)
+
+ def test_POST_meta_val_len(self):
+ with save_globals():
+ limit = constraints.MAX_META_VALUE_LENGTH
+ self.app.object_post_as_copy = False
+ ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ set_http_connect(200, 200, 202, 202, 202)
+ # acct cont obj obj obj
+ req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'},
+ headers={'Content-Type': 'foo/bar',
+ 'X-Object-Meta-Foo': 'x' * limit})
+ self.app.update_request(req)
+ res = req.get_response(self.app)
+ self.assertEqual(res.status_int, 202)
+ set_http_connect(202, 202, 202)
+ req = Request.blank(
+ '/v1/a/c/o', {'REQUEST_METHOD': 'POST'},
+ headers={'Content-Type': 'foo/bar',
+ 'X-Object-Meta-Foo': 'x' * (limit + 1)})
+ self.app.update_request(req)
+ res = req.get_response(self.app)
+ self.assertEqual(res.status_int, 400)
+
+ def test_POST_meta_authorize(self):
+ def authorize(req):
+ req.headers['X-Object-Meta-Foo'] = 'x' * (limit + 1)
+ return
+ with save_globals():
+ limit = constraints.MAX_META_VALUE_LENGTH
+ self.app.object_post_as_copy = False
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ set_http_connect(200, 200, 202, 202, 202)
+ # acct cont obj obj obj
+ req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'},
+ headers={'Content-Type': 'foo/bar',
+ 'X-Object-Meta-Foo': 'x'})
+ req.environ['swift.authorize'] = authorize
+ self.app.update_request(req)
+ res = controller.POST(req)
+ self.assertEqual(res.status_int, 400)
+
+ def test_POST_meta_key_len(self):
+ with save_globals():
+ limit = constraints.MAX_META_NAME_LENGTH
+ self.app.object_post_as_copy = False
+ set_http_connect(200, 200, 202, 202, 202)
+ # acct cont obj obj obj
+ req = Request.blank(
+ '/v1/a/c/o', {'REQUEST_METHOD': 'POST'},
+ headers={'Content-Type': 'foo/bar',
+ ('X-Object-Meta-' + 'x' * limit): 'x'})
+ self.app.update_request(req)
+ res = req.get_response(self.app)
+ self.assertEqual(res.status_int, 202)
+ set_http_connect(202, 202, 202)
+ req = Request.blank(
+ '/v1/a/c/o', {'REQUEST_METHOD': 'POST'},
+ headers={'Content-Type': 'foo/bar',
+ ('X-Object-Meta-' + 'x' * (limit + 1)): 'x'})
+ self.app.update_request(req)
+ res = req.get_response(self.app)
+ self.assertEqual(res.status_int, 400)
+
+ def test_POST_meta_count(self):
+ with save_globals():
+ limit = constraints.MAX_META_COUNT
+ headers = dict(
+ (('X-Object-Meta-' + str(i), 'a') for i in range(limit + 1)))
+ headers.update({'Content-Type': 'foo/bar'})
+ set_http_connect(202, 202, 202)
+ req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'},
+ headers=headers)
+ self.app.update_request(req)
+ res = req.get_response(self.app)
+ self.assertEqual(res.status_int, 400)
+
+ def test_POST_meta_size(self):
+ with save_globals():
+ limit = constraints.MAX_META_OVERALL_SIZE
+ count = limit / 256 # enough to cause the limit to be reached
+ headers = dict(
+ (('X-Object-Meta-' + str(i), 'a' * 256)
+ for i in range(count + 1)))
+ headers.update({'Content-Type': 'foo/bar'})
+ set_http_connect(202, 202, 202)
+ req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'POST'},
+ headers=headers)
+ self.app.update_request(req)
+ res = req.get_response(self.app)
+ self.assertEqual(res.status_int, 400)
+
+ def test_PUT_not_autodetect_content_type(self):
+ with save_globals():
+ headers = {'Content-Type': 'something/right', 'Content-Length': 0}
+ it_worked = []
+
+ def verify_content_type(ipaddr, port, device, partition,
+ method, path, headers=None,
+ query_string=None):
+ if path == '/a/c/o.html':
+ it_worked.append(
+ headers['Content-Type'].startswith('something/right'))
+
+ set_http_connect(204, 204, 201, 201, 201,
+ give_connect=verify_content_type)
+ req = Request.blank('/v1/a/c/o.html', {'REQUEST_METHOD': 'PUT'},
+ headers=headers)
+ self.app.update_request(req)
+ req.get_response(self.app)
+ self.assertNotEqual(it_worked, [])
+ self.assertTrue(all(it_worked))
+
+ def test_PUT_autodetect_content_type(self):
+ with save_globals():
+ headers = {'Content-Type': 'something/wrong', 'Content-Length': 0,
+ 'X-Detect-Content-Type': 'True'}
+ it_worked = []
+
+ def verify_content_type(ipaddr, port, device, partition,
+ method, path, headers=None,
+ query_string=None):
+ if path == '/a/c/o.html':
+ it_worked.append(
+ headers['Content-Type'].startswith('text/html'))
+
+ set_http_connect(204, 204, 201, 201, 201,
+ give_connect=verify_content_type)
+ req = Request.blank('/v1/a/c/o.html', {'REQUEST_METHOD': 'PUT'},
+ headers=headers)
+ self.app.update_request(req)
+ req.get_response(self.app)
+ self.assertNotEqual(it_worked, [])
+ self.assertTrue(all(it_worked))
+
+ def test_client_timeout(self):
+ with save_globals():
+ self.app.account_ring.get_nodes('account')
+ for dev in self.app.account_ring.devs:
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = 1
+ self.app.container_ring.get_nodes('account')
+ for dev in self.app.container_ring.devs:
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = 1
+ object_ring = self.app.get_object_ring(None)
+ object_ring.get_nodes('account')
+ for dev in object_ring.devs:
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = 1
+
+ class SlowBody(object):
+
+ def __init__(self):
+ self.sent = 0
+
+ def read(self, size=-1):
+ if self.sent < 4:
+ sleep(0.1)
+ self.sent += 1
+ return ' '
+ return ''
+
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'PUT',
+ 'wsgi.input': SlowBody()},
+ headers={'Content-Length': '4',
+ 'Content-Type': 'text/plain'})
+ self.app.update_request(req)
+ set_http_connect(200, 200, 201, 201, 201)
+ # acct cont obj obj obj
+ resp = req.get_response(self.app)
+ self.assertEqual(resp.status_int, 201)
+ self.app.client_timeout = 0.05
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'PUT',
+ 'wsgi.input': SlowBody()},
+ headers={'Content-Length': '4',
+ 'Content-Type': 'text/plain'})
+ self.app.update_request(req)
+ set_http_connect(201, 201, 201)
+ # obj obj obj
+ resp = req.get_response(self.app)
+ self.assertEqual(resp.status_int, 408)
+
+ def test_client_disconnect(self):
+ with save_globals():
+ self.app.account_ring.get_nodes('account')
+ for dev in self.app.account_ring.devs:
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = 1
+ self.app.container_ring.get_nodes('account')
+ for dev in self.app.container_ring.devs:
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = 1
+ object_ring = self.app.get_object_ring(None)
+ object_ring.get_nodes('account')
+ for dev in object_ring.devs:
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = 1
+
+ class DisconnectedBody(object):
+
+ def __init__(self):
+ self.sent = 0
+
+ def read(self, size=-1):
+ return ''
+
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'PUT',
+ 'wsgi.input': DisconnectedBody()},
+ headers={'Content-Length': '4',
+ 'Content-Type': 'text/plain'})
+ self.app.update_request(req)
+ set_http_connect(200, 200, 201, 201, 201)
+ # acct cont obj obj obj
+ resp = req.get_response(self.app)
+ self.assertEqual(resp.status_int, 499)
+
+ def test_node_read_timeout(self):
+ with save_globals():
+ self.app.account_ring.get_nodes('account')
+ for dev in self.app.account_ring.devs:
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = 1
+ self.app.container_ring.get_nodes('account')
+ for dev in self.app.container_ring.devs:
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = 1
+ object_ring = self.app.get_object_ring(None)
+ object_ring.get_nodes('account')
+ for dev in object_ring.devs:
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = 1
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
+ self.app.update_request(req)
+ set_http_connect(200, 200, 200, slow=0.1)
+ req.sent_size = 0
+ resp = req.get_response(self.app)
+ got_exc = False
+ try:
+ resp.body
+ except ChunkReadTimeout:
+ got_exc = True
+ self.assertFalse(got_exc)
+ self.app.recoverable_node_timeout = 0.1
+ set_http_connect(200, 200, 200, slow=1.0)
+ resp = req.get_response(self.app)
+ got_exc = False
+ try:
+ resp.body
+ except ChunkReadTimeout:
+ got_exc = True
+ self.assertTrue(got_exc)
+
+ def test_node_read_timeout_retry(self):
+ with save_globals():
+ self.app.account_ring.get_nodes('account')
+ for dev in self.app.account_ring.devs:
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = 1
+ self.app.container_ring.get_nodes('account')
+ for dev in self.app.container_ring.devs:
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = 1
+ object_ring = self.app.get_object_ring(None)
+ object_ring.get_nodes('account')
+ for dev in object_ring.devs:
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = 1
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
+ self.app.update_request(req)
+
+ self.app.recoverable_node_timeout = 0.1
+ set_http_connect(200, 200, 200, slow=[1.0, 1.0, 1.0])
+ resp = req.get_response(self.app)
+ got_exc = False
+ try:
+ self.assertEqual('', resp.body)
+ except ChunkReadTimeout:
+ got_exc = True
+ self.assertTrue(got_exc)
+
+ set_http_connect(200, 200, 200, body='lalala',
+ slow=[1.0, 1.0])
+ resp = req.get_response(self.app)
+ got_exc = False
+ try:
+ self.assertEqual(resp.body, 'lalala')
+ except ChunkReadTimeout:
+ got_exc = True
+ self.assertFalse(got_exc)
+
+ set_http_connect(200, 200, 200, body='lalala',
+ slow=[1.0, 1.0], etags=['a', 'a', 'a'])
+ resp = req.get_response(self.app)
+ got_exc = False
+ try:
+ self.assertEqual(resp.body, 'lalala')
+ except ChunkReadTimeout:
+ got_exc = True
+ self.assertFalse(got_exc)
+
+ set_http_connect(200, 200, 200, body='lalala',
+ slow=[1.0, 1.0], etags=['a', 'b', 'a'])
+ resp = req.get_response(self.app)
+ got_exc = False
+ try:
+ self.assertEqual(resp.body, 'lalala')
+ except ChunkReadTimeout:
+ got_exc = True
+ self.assertFalse(got_exc)
+
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
+ set_http_connect(200, 200, 200, body='lalala',
+ slow=[1.0, 1.0], etags=['a', 'b', 'b'])
+ resp = req.get_response(self.app)
+ got_exc = False
+ try:
+ resp.body
+ except ChunkReadTimeout:
+ got_exc = True
+ self.assertTrue(got_exc)
+
+ def test_node_write_timeout(self):
+ with save_globals():
+ self.app.account_ring.get_nodes('account')
+ for dev in self.app.account_ring.devs:
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = 1
+ self.app.container_ring.get_nodes('account')
+ for dev in self.app.container_ring.devs:
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = 1
+ object_ring = self.app.get_object_ring(None)
+ object_ring.get_nodes('account')
+ for dev in object_ring.devs:
+ dev['ip'] = '127.0.0.1'
+ dev['port'] = 1
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'PUT'},
+ headers={'Content-Length': '4',
+ 'Content-Type': 'text/plain'},
+ body=' ')
+ self.app.update_request(req)
+ set_http_connect(200, 200, 201, 201, 201, slow=0.1)
+ resp = req.get_response(self.app)
+ self.assertEqual(resp.status_int, 201)
+ self.app.node_timeout = 0.1
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'PUT'},
+ headers={'Content-Length': '4',
+ 'Content-Type': 'text/plain'},
+ body=' ')
+ self.app.update_request(req)
+ set_http_connect(201, 201, 201, slow=1.0)
+ resp = req.get_response(self.app)
+ self.assertEqual(resp.status_int, 503)
+
+ def test_node_request_setting(self):
+ baseapp = proxy_server.Application({'request_node_count': '3'},
+ FakeMemcache(),
+ container_ring=FakeRing(),
+ account_ring=FakeRing())
+ self.assertEqual(baseapp.request_node_count(3), 3)
+
+ def test_iter_nodes(self):
+ with save_globals():
+ try:
+ object_ring = self.app.get_object_ring(None)
+ object_ring.max_more_nodes = 2
+ partition, nodes = object_ring.get_nodes('account',
+ 'container',
+ 'object')
+ collected_nodes = []
+ for node in self.app.iter_nodes(object_ring,
+ partition):
+ collected_nodes.append(node)
+ self.assertEqual(len(collected_nodes), 5)
+
+ object_ring.max_more_nodes = 6
+ self.app.request_node_count = lambda r: 20
+ partition, nodes = object_ring.get_nodes('account',
+ 'container',
+ 'object')
+ collected_nodes = []
+ for node in self.app.iter_nodes(object_ring,
+ partition):
+ collected_nodes.append(node)
+ self.assertEqual(len(collected_nodes), 9)
+
+ # zero error-limited primary nodes -> no handoff warnings
+ self.app.log_handoffs = True
+ self.app.logger = FakeLogger()
+ self.app.request_node_count = lambda r: 7
+ object_ring.max_more_nodes = 20
+ partition, nodes = object_ring.get_nodes('account',
+ 'container',
+ 'object')
+ collected_nodes = []
+ for node in self.app.iter_nodes(object_ring, partition):
+ collected_nodes.append(node)
+ self.assertEqual(len(collected_nodes), 7)
+ self.assertEqual(self.app.logger.log_dict['warning'], [])
+ self.assertEqual(self.app.logger.get_increments(), [])
+
+ # one error-limited primary node -> one handoff warning
+ self.app.log_handoffs = True
+ self.app.logger = FakeLogger()
+ self.app.request_node_count = lambda r: 7
+ self.app._error_limiting = {} # clear out errors
+ set_node_errors(self.app, object_ring._devs[0], 999,
+ last_error=(2 ** 63 - 1))
+
+ collected_nodes = []
+ for node in self.app.iter_nodes(object_ring, partition):
+ collected_nodes.append(node)
+ self.assertEqual(len(collected_nodes), 7)
+ self.assertEqual(self.app.logger.log_dict['warning'], [
+ (('Handoff requested (5)',), {})])
+ self.assertEqual(self.app.logger.get_increments(),
+ ['handoff_count'])
+
+ # two error-limited primary nodes -> two handoff warnings
+ self.app.log_handoffs = True
+ self.app.logger = FakeLogger()
+ self.app.request_node_count = lambda r: 7
+ self.app._error_limiting = {} # clear out errors
+ for i in range(2):
+ set_node_errors(self.app, object_ring._devs[i], 999,
+ last_error=(2 ** 63 - 1))
+
+ collected_nodes = []
+ for node in self.app.iter_nodes(object_ring, partition):
+ collected_nodes.append(node)
+ self.assertEqual(len(collected_nodes), 7)
+ self.assertEqual(self.app.logger.log_dict['warning'], [
+ (('Handoff requested (5)',), {}),
+ (('Handoff requested (6)',), {})])
+ self.assertEqual(self.app.logger.get_increments(),
+ ['handoff_count',
+ 'handoff_count'])
+
+ # all error-limited primary nodes -> four handoff warnings,
+ # plus a handoff-all metric
+ self.app.log_handoffs = True
+ self.app.logger = FakeLogger()
+ self.app.request_node_count = lambda r: 10
+ object_ring.set_replicas(4) # otherwise we run out of handoffs
+ self.app._error_limiting = {} # clear out errors
+ for i in range(4):
+ set_node_errors(self.app, object_ring._devs[i], 999,
+ last_error=(2 ** 63 - 1))
+
+ collected_nodes = []
+ for node in self.app.iter_nodes(object_ring, partition):
+ collected_nodes.append(node)
+ self.assertEqual(len(collected_nodes), 10)
+ self.assertEqual(self.app.logger.log_dict['warning'], [
+ (('Handoff requested (7)',), {}),
+ (('Handoff requested (8)',), {}),
+ (('Handoff requested (9)',), {}),
+ (('Handoff requested (10)',), {})])
+ self.assertEqual(self.app.logger.get_increments(),
+ ['handoff_count',
+ 'handoff_count',
+ 'handoff_count',
+ 'handoff_count',
+ 'handoff_all_count'])
+
+ finally:
+ object_ring.max_more_nodes = 0
+
+ def test_iter_nodes_calls_sort_nodes(self):
+ with mock.patch.object(self.app, 'sort_nodes') as sort_nodes:
+ object_ring = self.app.get_object_ring(None)
+ for node in self.app.iter_nodes(object_ring, 0):
+ pass
+ sort_nodes.assert_called_once_with(
+ object_ring.get_part_nodes(0))
+
+ def test_iter_nodes_skips_error_limited(self):
+ with mock.patch.object(self.app, 'sort_nodes', lambda n: n):
+ object_ring = self.app.get_object_ring(None)
+ first_nodes = list(self.app.iter_nodes(object_ring, 0))
+ second_nodes = list(self.app.iter_nodes(object_ring, 0))
+ self.assertIn(first_nodes[0], second_nodes)
+
+ self.app.error_limit(first_nodes[0], 'test')
+ second_nodes = list(self.app.iter_nodes(object_ring, 0))
+ self.assertNotIn(first_nodes[0], second_nodes)
+
+ def test_iter_nodes_gives_extra_if_error_limited_inline(self):
+ object_ring = self.app.get_object_ring(None)
+ with mock.patch.object(self.app, 'sort_nodes', lambda n: n), \
+ mock.patch.object(self.app, 'request_node_count',
+ lambda r: 6), \
+ mock.patch.object(object_ring, 'max_more_nodes', 99):
+ first_nodes = list(self.app.iter_nodes(object_ring, 0))
+ second_nodes = []
+ for node in self.app.iter_nodes(object_ring, 0):
+ if not second_nodes:
+ self.app.error_limit(node, 'test')
+ second_nodes.append(node)
+ self.assertEqual(len(first_nodes), 6)
+ self.assertEqual(len(second_nodes), 7)
+
+ def test_iter_nodes_with_custom_node_iter(self):
+ object_ring = self.app.get_object_ring(None)
+ node_list = [dict(id=n, ip='1.2.3.4', port=n, device='D')
+ for n in range(10)]
+ with mock.patch.object(self.app, 'sort_nodes', lambda n: n), \
+ mock.patch.object(self.app, 'request_node_count',
+ lambda r: 3):
+ got_nodes = list(self.app.iter_nodes(object_ring, 0,
+ node_iter=iter(node_list)))
+ self.assertEqual(node_list[:3], got_nodes)
+
+ with mock.patch.object(self.app, 'sort_nodes', lambda n: n), \
+ mock.patch.object(self.app, 'request_node_count',
+ lambda r: 1000000):
+ got_nodes = list(self.app.iter_nodes(object_ring, 0,
+ node_iter=iter(node_list)))
+ self.assertEqual(node_list, got_nodes)
+
+ def test_best_response_sets_headers(self):
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
+ resp = controller.best_response(req, [200] * 3, ['OK'] * 3, [''] * 3,
+ 'Object', headers=[{'X-Test': '1'},
+ {'X-Test': '2'},
+ {'X-Test': '3'}])
+ self.assertEqual(resp.headers['X-Test'], '1')
+
+ def test_best_response_sets_etag(self):
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
+ resp = controller.best_response(req, [200] * 3, ['OK'] * 3, [''] * 3,
+ 'Object')
+ self.assertIsNone(resp.etag)
+ resp = controller.best_response(req, [200] * 3, ['OK'] * 3, [''] * 3,
+ 'Object',
+ etag='68b329da9893e34099c7d8ad5cb9c940'
+ )
+ self.assertEqual(resp.etag, '68b329da9893e34099c7d8ad5cb9c940')
+
+ def test_proxy_passes_content_type(self):
+ with save_globals():
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
+ self.app.update_request(req)
+ set_http_connect(200, 200, 200)
+ resp = req.get_response(self.app)
+ self.assertEqual(resp.status_int, 200)
+ self.assertEqual(resp.content_type, 'x-application/test')
+ set_http_connect(200, 200, 200)
+ resp = req.get_response(self.app)
+ self.assertEqual(resp.status_int, 200)
+ self.assertEqual(resp.content_length, 0)
+ set_http_connect(200, 200, 200, slow=True)
+ resp = req.get_response(self.app)
+ self.assertEqual(resp.status_int, 200)
+ self.assertEqual(resp.content_length, 4)
+
+ def test_proxy_passes_content_length_on_head(self):
+ with save_globals():
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'HEAD'})
+ self.app.update_request(req)
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ set_http_connect(200, 200, 200)
+ resp = controller.HEAD(req)
+ self.assertEqual(resp.status_int, 200)
+ self.assertEqual(resp.content_length, 0)
+ set_http_connect(200, 200, 200, slow=True)
+ resp = controller.HEAD(req)
+ self.assertEqual(resp.status_int, 200)
+ self.assertEqual(resp.content_length, 4)
+
+ def test_error_limiting(self):
+ with save_globals():
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ controller.app.sort_nodes = lambda l: l
+ object_ring = controller.app.get_object_ring(None)
+ self.assert_status_map(controller.HEAD, (200, 200, 503, 200, 200),
+ 200)
+ self.assertEqual(
+ node_error_count(controller.app, object_ring.devs[0]), 2)
+ self.assertTrue(
+ node_last_error(controller.app, object_ring.devs[0])
+ is not None)
+ for _junk in range(self.app.error_suppression_limit):
+ self.assert_status_map(controller.HEAD, (200, 200, 503, 503,
+ 503), 503)
+ self.assertEqual(
+ node_error_count(controller.app, object_ring.devs[0]),
+ self.app.error_suppression_limit + 1)
+ self.assert_status_map(controller.HEAD, (200, 200, 200, 200, 200),
+ 503)
+ self.assertTrue(
+ node_last_error(controller.app, object_ring.devs[0])
+ is not None)
+ self.assert_status_map(controller.PUT, (200, 200, 200, 201, 201,
+ 201), 503)
+ self.assert_status_map(controller.POST,
+ (200, 200, 200, 200, 200, 200, 202, 202,
+ 202), 503)
+ self.assert_status_map(controller.DELETE,
+ (200, 200, 200, 204, 204, 204), 503)
+ self.app.error_suppression_interval = -300
+ self.assert_status_map(controller.HEAD, (200, 200, 200, 200, 200),
+ 200)
+ self.assertRaises(BaseException,
+ self.assert_status_map, controller.DELETE,
+ (200, 200, 200, 204, 204, 204), 503,
+ raise_exc=True)
+
+ def test_error_limiting_survives_ring_reload(self):
+ with save_globals():
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ controller.app.sort_nodes = lambda l: l
+ object_ring = controller.app.get_object_ring(None)
+ self.assert_status_map(controller.HEAD, (200, 200, 503, 200, 200),
+ 200)
+ self.assertEqual(
+ node_error_count(controller.app, object_ring.devs[0]), 2)
+ self.assertTrue(
+ node_last_error(controller.app, object_ring.devs[0])
+ is not None)
+ for _junk in range(self.app.error_suppression_limit):
+ self.assert_status_map(controller.HEAD, (200, 200, 503, 503,
+ 503), 503)
+ self.assertEqual(
+ node_error_count(controller.app, object_ring.devs[0]),
+ self.app.error_suppression_limit + 1)
+
+ # wipe out any state in the ring
+ for policy in POLICIES:
+ policy.object_ring = FakeRing(base_port=3000)
+
+ # and we still get an error, which proves that the
+ # error-limiting info survived a ring reload
+ self.assert_status_map(controller.HEAD, (200, 200, 200, 200, 200),
+ 503)
+
+ def test_PUT_error_limiting(self):
+ with save_globals():
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ controller.app.sort_nodes = lambda l: l
+ object_ring = controller.app.get_object_ring(None)
+ # acc con obj obj obj
+ self.assert_status_map(controller.PUT, (200, 200, 503, 200, 200),
+ 200)
+
+ # 2, not 1, because assert_status_map() calls the method twice
+ odevs = object_ring.devs
+ self.assertEqual(node_error_count(controller.app, odevs[0]), 2)
+ self.assertEqual(node_error_count(controller.app, odevs[1]), 0)
+ self.assertEqual(node_error_count(controller.app, odevs[2]), 0)
+ self.assertTrue(
+ node_last_error(controller.app, odevs[0]) is not None)
+ self.assertTrue(node_last_error(controller.app, odevs[1]) is None)
+ self.assertTrue(node_last_error(controller.app, odevs[2]) is None)
+
+ def test_PUT_error_limiting_last_node(self):
+ with save_globals():
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ controller.app.sort_nodes = lambda l: l
+ object_ring = controller.app.get_object_ring(None)
+ # acc con obj obj obj
+ self.assert_status_map(controller.PUT, (200, 200, 200, 200, 503),
+ 200)
+
+ # 2, not 1, because assert_status_map() calls the method twice
+ odevs = object_ring.devs
+ self.assertEqual(node_error_count(controller.app, odevs[0]), 0)
+ self.assertEqual(node_error_count(controller.app, odevs[1]), 0)
+ self.assertEqual(node_error_count(controller.app, odevs[2]), 2)
+ self.assertTrue(node_last_error(controller.app, odevs[0]) is None)
+ self.assertTrue(node_last_error(controller.app, odevs[1]) is None)
+ self.assertTrue(
+ node_last_error(controller.app, odevs[2]) is not None)
+
+ def test_acc_or_con_missing_returns_404(self):
+ with save_globals():
+ self.app.memcache = FakeMemcacheReturnsNone()
+ self.app._error_limiting = {}
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ set_http_connect(200, 200, 200, 200, 200, 200)
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'DELETE'})
+ self.app.update_request(req)
+ resp = getattr(controller, 'DELETE')(req)
+ self.assertEqual(resp.status_int, 200)
+
+ set_http_connect(404, 404, 404)
+ # acct acct acct
+ # make sure to use a fresh request without cached env
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'DELETE'})
+ resp = getattr(controller, 'DELETE')(req)
+ self.assertEqual(resp.status_int, 404)
+
+ set_http_connect(503, 404, 404)
+ # acct acct acct
+ # make sure to use a fresh request without cached env
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'DELETE'})
+ resp = getattr(controller, 'DELETE')(req)
+ self.assertEqual(resp.status_int, 404)
+
+ set_http_connect(503, 503, 404)
+ # acct acct acct
+ # make sure to use a fresh request without cached env
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'DELETE'})
+ resp = getattr(controller, 'DELETE')(req)
+ self.assertEqual(resp.status_int, 404)
+
+ set_http_connect(503, 503, 503)
+ # acct acct acct
+ # make sure to use a fresh request without cached env
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'DELETE'})
+ resp = getattr(controller, 'DELETE')(req)
+ self.assertEqual(resp.status_int, 404)
+
+ set_http_connect(200, 200, 204, 204, 204)
+ # acct cont obj obj obj
+ # make sure to use a fresh request without cached env
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'DELETE'})
+ resp = getattr(controller, 'DELETE')(req)
+ self.assertEqual(resp.status_int, 204)
+
+ set_http_connect(200, 404, 404, 404)
+ # acct cont cont cont
+ # make sure to use a fresh request without cached env
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'DELETE'})
+ resp = getattr(controller, 'DELETE')(req)
+ self.assertEqual(resp.status_int, 404)
+
+ set_http_connect(200, 503, 503, 503)
+ # acct cont cont cont
+ # make sure to use a fresh request without cached env
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'DELETE'})
+ resp = getattr(controller, 'DELETE')(req)
+ self.assertEqual(resp.status_int, 404)
+
+ for dev in self.app.account_ring.devs:
+ set_node_errors(
+ self.app, dev, self.app.error_suppression_limit + 1,
+ time.time())
+ set_http_connect(200)
+ # acct [isn't actually called since everything
+ # is error limited]
+ # make sure to use a fresh request without cached env
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'DELETE'})
+ resp = getattr(controller, 'DELETE')(req)
+ self.assertEqual(resp.status_int, 404)
+
+ for dev in self.app.account_ring.devs:
+ set_node_errors(self.app, dev, 0, last_error=None)
+ for dev in self.app.container_ring.devs:
+ set_node_errors(self.app, dev,
+ self.app.error_suppression_limit + 1,
+ time.time())
+ set_http_connect(200, 200)
+ # acct cont [isn't actually called since
+ # everything is error limited]
+ # make sure to use a fresh request without cached env
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'DELETE'})
+ resp = getattr(controller, 'DELETE')(req)
+ self.assertEqual(resp.status_int, 404)
+
+ def test_PUT_POST_requires_container_exist(self):
+ with save_globals():
+ self.app.object_post_as_copy = False
+ self.app.memcache = FakeMemcacheReturnsNone()
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+
+ set_http_connect(200, 404, 404, 404, 200, 200, 200)
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'PUT'})
+ self.app.update_request(req)
+ resp = controller.PUT(req)
+ self.assertEqual(resp.status_int, 404)
+
+ set_http_connect(200, 404, 404, 404, 200, 200)
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'POST'},
+ headers={'Content-Type': 'text/plain'})
+ self.app.update_request(req)
+ resp = controller.POST(req)
+ self.assertEqual(resp.status_int, 404)
+
+ def test_bad_metadata(self):
+ with save_globals():
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ set_http_connect(200, 200, 201, 201, 201)
+ # acct cont obj obj obj
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+ headers={'Content-Length': '0'})
+ self.app.update_request(req)
+ resp = controller.PUT(req)
+ self.assertEqual(resp.status_int, 201)
+
+ set_http_connect(201, 201, 201)
+ req = Request.blank(
+ '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+ headers={'Content-Length': '0',
+ 'X-Object-Meta-' + (
+ 'a' * constraints.MAX_META_NAME_LENGTH): 'v'})
+ self.app.update_request(req)
+ resp = controller.PUT(req)
+ self.assertEqual(resp.status_int, 201)
+ set_http_connect(201, 201, 201)
+ req = Request.blank(
+ '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+ headers={
+ 'Content-Length': '0',
+ 'X-Object-Meta-' + (
+ 'a' * (constraints.MAX_META_NAME_LENGTH + 1)): 'v'})
+ self.app.update_request(req)
+ resp = controller.PUT(req)
+ self.assertEqual(resp.status_int, 400)
+
+ set_http_connect(201, 201, 201)
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+ headers={'Content-Length': '0',
+ 'X-Object-Meta-Too-Long': 'a' *
+ constraints.MAX_META_VALUE_LENGTH})
+ self.app.update_request(req)
+ resp = controller.PUT(req)
+ self.assertEqual(resp.status_int, 201)
+ set_http_connect(201, 201, 201)
+ req = Request.blank(
+ '/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+ headers={'Content-Length': '0',
+ 'X-Object-Meta-Too-Long': 'a' *
+ (constraints.MAX_META_VALUE_LENGTH + 1)})
+ self.app.update_request(req)
+ resp = controller.PUT(req)
+ self.assertEqual(resp.status_int, 400)
+
+ set_http_connect(201, 201, 201)
+ headers = {'Content-Length': '0'}
+ for x in range(constraints.MAX_META_COUNT):
+ headers['X-Object-Meta-%d' % x] = 'v'
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+ headers=headers)
+ self.app.update_request(req)
+ resp = controller.PUT(req)
+ self.assertEqual(resp.status_int, 201)
+ set_http_connect(201, 201, 201)
+ headers = {'Content-Length': '0'}
+ for x in range(constraints.MAX_META_COUNT + 1):
+ headers['X-Object-Meta-%d' % x] = 'v'
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+ headers=headers)
+ self.app.update_request(req)
+ resp = controller.PUT(req)
+ self.assertEqual(resp.status_int, 400)
+
+ set_http_connect(201, 201, 201)
+ headers = {'Content-Length': '0'}
+ header_value = 'a' * constraints.MAX_META_VALUE_LENGTH
+ size = 0
+ x = 0
+ while size < constraints.MAX_META_OVERALL_SIZE - 4 - \
+ constraints.MAX_META_VALUE_LENGTH:
+ size += 4 + constraints.MAX_META_VALUE_LENGTH
+ headers['X-Object-Meta-%04d' % x] = header_value
+ x += 1
+ if constraints.MAX_META_OVERALL_SIZE - size > 1:
+ headers['X-Object-Meta-a'] = \
+ 'a' * (constraints.MAX_META_OVERALL_SIZE - size - 1)
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+ headers=headers)
+ self.app.update_request(req)
+ resp = controller.PUT(req)
+ self.assertEqual(resp.status_int, 201)
+ set_http_connect(201, 201, 201)
+ headers['X-Object-Meta-a'] = \
+ 'a' * (constraints.MAX_META_OVERALL_SIZE - size)
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+ headers=headers)
+ self.app.update_request(req)
+ resp = controller.PUT(req)
+ self.assertEqual(resp.status_int, 400)
+
+ @contextmanager
+ def controller_context(self, req, *args, **kwargs):
+ _v, account, container, obj = utils.split_path(req.path, 4, 4, True)
+ controller = ReplicatedObjectController(
+ self.app, account, container, obj)
+ self.app.update_request(req)
+ self.app.memcache.store = {}
+ with save_globals():
+ new_connect = set_http_connect(*args, **kwargs)
+ yield controller
+ unused_status_list = []
+ while True:
+ try:
+ unused_status_list.append(next(new_connect.code_iter))
+ except StopIteration:
+ break
+ if unused_status_list:
+ raise self.fail('UN-USED STATUS CODES: %r' %
+ unused_status_list)
+
+ def test_mismatched_etags(self):
+ with save_globals():
+ # no etag supplied, object servers return success w/ diff values
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+ headers={'Content-Length': '0'})
+ self.app.update_request(req)
+ set_http_connect(200, 201, 201, 201,
+ etags=[None,
+ '68b329da9893e34099c7d8ad5cb9c940',
+ '68b329da9893e34099c7d8ad5cb9c940',
+ '68b329da9893e34099c7d8ad5cb9c941'])
+ resp = controller.PUT(req)
+ self.assertEqual(resp.status_int // 100, 5) # server error
+
+ # req supplies etag, object servers return 422 - mismatch
+ headers = {'Content-Length': '0',
+ 'ETag': '68b329da9893e34099c7d8ad5cb9c940'}
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+ headers=headers)
+ self.app.update_request(req)
+ set_http_connect(200, 422, 422, 503,
+ etags=['68b329da9893e34099c7d8ad5cb9c940',
+ '68b329da9893e34099c7d8ad5cb9c941',
+ None,
+ None])
+ resp = controller.PUT(req)
+ self.assertEqual(resp.status_int // 100, 4) # client error
+
+ def test_response_get_accept_ranges_header(self):
+ with save_globals():
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
+ self.app.update_request(req)
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ set_http_connect(200, 200, 200)
+ resp = controller.GET(req)
+ self.assertIn('accept-ranges', resp.headers)
+ self.assertEqual(resp.headers['accept-ranges'], 'bytes')
+
+ def test_response_head_accept_ranges_header(self):
+ with save_globals():
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'HEAD'})
+ self.app.update_request(req)
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ set_http_connect(200, 200, 200)
+ resp = controller.HEAD(req)
+ self.assertIn('accept-ranges', resp.headers)
+ self.assertEqual(resp.headers['accept-ranges'], 'bytes')
+
+ def test_GET_calls_authorize(self):
+ called = [False]
+
+ def authorize(req):
+ called[0] = True
+ return HTTPUnauthorized(request=req)
+ with save_globals():
+ set_http_connect(200, 200, 201, 201, 201)
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ req = Request.blank('/v1/a/c/o')
+ req.environ['swift.authorize'] = authorize
+ self.app.update_request(req)
+ controller.GET(req)
+ self.assertTrue(called[0])
+
+ def test_HEAD_calls_authorize(self):
+ called = [False]
+
+ def authorize(req):
+ called[0] = True
+ return HTTPUnauthorized(request=req)
+ with save_globals():
+ set_http_connect(200, 200, 201, 201, 201)
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ req = Request.blank('/v1/a/c/o', {'REQUEST_METHOD': 'HEAD'})
+ req.environ['swift.authorize'] = authorize
+ self.app.update_request(req)
+ controller.HEAD(req)
+ self.assertTrue(called[0])
+
+ def test_POST_calls_authorize(self):
+ called = [False]
+
+ def authorize(req):
+ called[0] = True
+ return HTTPUnauthorized(request=req)
+ with save_globals():
+ self.app.object_post_as_copy = False
+ set_http_connect(200, 200, 201, 201, 201)
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'POST'},
+ headers={'Content-Length': '5'}, body='12345')
+ req.environ['swift.authorize'] = authorize
+ self.app.update_request(req)
+ controller.POST(req)
+ self.assertTrue(called[0])
+
+ def test_PUT_calls_authorize(self):
+ called = [False]
+
+ def authorize(req):
+ called[0] = True
+ return HTTPUnauthorized(request=req)
+ with save_globals():
+ set_http_connect(200, 200, 201, 201, 201)
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+ headers={'Content-Length': '5'}, body='12345')
+ req.environ['swift.authorize'] = authorize
+ self.app.update_request(req)
+ controller.PUT(req)
+ self.assertTrue(called[0])
+
+ def test_POST_converts_delete_after_to_delete_at(self):
+ with save_globals():
+ self.app.object_post_as_copy = False
+ controller = ReplicatedObjectController(
+ self.app, 'account', 'container', 'object')
+ set_http_connect(200, 200, 202, 202, 202)
+ self.app.memcache.store = {}
+ orig_time = time.time
+ try:
+ t = time.time()
+ time.time = lambda: t
+ req = Request.blank('/v1/a/c/o', {},
+ headers={'Content-Type': 'foo/bar',
+ 'X-Delete-After': '60'})
+ self.app.update_request(req)
+ res = controller.POST(req)
+ self.assertEqual(res.status, '202 Fake')
+ self.assertEqual(req.headers.get('x-delete-at'),
+ str(int(t + 60)))
+ finally:
+ time.time = orig_time
+
+ def test_OPTIONS(self):
+ with save_globals():
+ controller = ReplicatedObjectController(
+ self.app, 'a', 'c', 'o.jpg')
+
+ def my_empty_container_info(*args):
+ return {}
+ controller.container_info = my_empty_container_info
+ req = Request.blank(
+ '/v1/a/c/o.jpg',
+ {'REQUEST_METHOD': 'OPTIONS'},
+ headers={'Origin': 'http://foo.com',
+ 'Access-Control-Request-Method': 'GET'})
+ resp = controller.OPTIONS(req)
+ self.assertEqual(401, resp.status_int)
+
+ def my_empty_origin_container_info(*args):
+ return {'cors': {'allow_origin': None}}
+ controller.container_info = my_empty_origin_container_info
+ req = Request.blank(
+ '/v1/a/c/o.jpg',
+ {'REQUEST_METHOD': 'OPTIONS'},
+ headers={'Origin': 'http://foo.com',
+ 'Access-Control-Request-Method': 'GET'})
+ resp = controller.OPTIONS(req)
+ self.assertEqual(401, resp.status_int)
+
+ def my_container_info(*args):
+ return {
+ 'cors': {
+ 'allow_origin': 'http://foo.bar:8080 https://foo.bar',
+ 'max_age': '999',
+ }
+ }
+ controller.container_info = my_container_info
+ req = Request.blank(
+ '/v1/a/c/o.jpg',
+ {'REQUEST_METHOD': 'OPTIONS'},
+ headers={'Origin': 'https://foo.bar',
+ 'Access-Control-Request-Method': 'GET'})
+ req.content_length = 0
+ resp = controller.OPTIONS(req)
+ self.assertEqual(200, resp.status_int)
+ self.assertEqual(
+ 'https://foo.bar',
+ resp.headers['access-control-allow-origin'])
+ for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split():
+ self.assertIn(verb,
+ resp.headers['access-control-allow-methods'])
+ self.assertEqual(
+ len(resp.headers['access-control-allow-methods'].split(', ')),
+ 6)
+ self.assertEqual('999', resp.headers['access-control-max-age'])
+ req = Request.blank(
+ '/v1/a/c/o.jpg',
+ {'REQUEST_METHOD': 'OPTIONS'},
+ headers={'Origin': 'https://foo.bar'})
+ req.content_length = 0
+ resp = controller.OPTIONS(req)
+ self.assertEqual(401, resp.status_int)
+ req = Request.blank('/v1/a/c/o.jpg', {'REQUEST_METHOD': 'OPTIONS'})
+ req.content_length = 0
+ resp = controller.OPTIONS(req)
+ self.assertEqual(200, resp.status_int)
+ for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split():
+ self.assertIn(verb, resp.headers['Allow'])
+ self.assertEqual(len(resp.headers['Allow'].split(', ')), 6)
+ req = Request.blank(
+ '/v1/a/c/o.jpg',
+ {'REQUEST_METHOD': 'OPTIONS'},
+ headers={'Origin': 'http://foo.com'})
+ resp = controller.OPTIONS(req)
+ self.assertEqual(401, resp.status_int)
+ req = Request.blank(
+ '/v1/a/c/o.jpg',
+ {'REQUEST_METHOD': 'OPTIONS'},
+ headers={'Origin': 'http://foo.bar',
+ 'Access-Control-Request-Method': 'GET'})
+ controller.app.cors_allow_origin = ['http://foo.bar', ]
+ resp = controller.OPTIONS(req)
+ self.assertEqual(200, resp.status_int)
+
+ def my_container_info_wildcard(*args):
+ return {
+ 'cors': {
+ 'allow_origin': '*',
+ 'max_age': '999',
+ }
+ }
+ controller.container_info = my_container_info_wildcard
+ req = Request.blank(
+ '/v1/a/c/o.jpg',
+ {'REQUEST_METHOD': 'OPTIONS'},
+ headers={'Origin': 'https://bar.baz',
+ 'Access-Control-Request-Method': 'GET'})
+ req.content_length = 0
+ resp = controller.OPTIONS(req)
+ self.assertEqual(200, resp.status_int)
+ self.assertEqual('*', resp.headers['access-control-allow-origin'])
+ for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split():
+ self.assertIn(verb,
+ resp.headers['access-control-allow-methods'])
+ self.assertEqual(
+ len(resp.headers['access-control-allow-methods'].split(', ')),
+ 6)
+ self.assertEqual('999', resp.headers['access-control-max-age'])
+
+ def _get_CORS_response(self, container_cors, strict_mode, object_get=None):
+ with save_globals():
+ controller = ReplicatedObjectController(
+ self.app, 'a', 'c', 'o')
+
+ def stubContainerInfo(*args):
+ return {
+ 'cors': container_cors
+ }
+
+ controller.container_info = stubContainerInfo
+ controller.app.strict_cors_mode = strict_mode
+
+ def objectGET(controller, req):
+ return Response(headers={
+ 'X-Object-Meta-Color': 'red',
+ 'X-Super-Secret': 'hush',
+ })
+
+ mock_object_get = object_get or objectGET
+
+ req = Request.blank(
+ '/v1/a/c/o.jpg',
+ {'REQUEST_METHOD': 'GET'},
+ headers={'Origin': 'http://foo.bar'})
+
+ resp = cors_validation(mock_object_get)(controller, req)
+
+ return resp
+
+ def test_CORS_valid_non_strict(self):
+ # test expose_headers to non-allowed origins
+ container_cors = {'allow_origin': 'http://not.foo.bar',
+ 'expose_headers': 'X-Object-Meta-Color '
+ 'X-Object-Meta-Color-Ex'}
+ resp = self._get_CORS_response(
+ container_cors=container_cors, strict_mode=False)
+
+ self.assertEqual(200, resp.status_int)
+ self.assertEqual('http://foo.bar',
+ resp.headers['access-control-allow-origin'])
+ self.assertEqual('red', resp.headers['x-object-meta-color'])
+ # X-Super-Secret is in the response, but not "exposed"
+ self.assertEqual('hush', resp.headers['x-super-secret'])
+ self.assertIn('access-control-expose-headers', resp.headers)
+ exposed = set(
+ h.strip() for h in
+ resp.headers['access-control-expose-headers'].split(','))
+ expected_exposed = set([
+ 'cache-control', 'content-language', 'content-type', 'expires',
+ 'last-modified', 'pragma', 'etag', 'x-timestamp', 'x-trans-id',
+ 'x-openstack-request-id', 'x-object-meta-color',
+ 'x-object-meta-color-ex'])
+ self.assertEqual(expected_exposed, exposed)
+
+ # test allow_origin *
+ container_cors = {'allow_origin': '*'}
+
+ resp = self._get_CORS_response(
+ container_cors=container_cors, strict_mode=False)
+ self.assertEqual(200, resp.status_int)
+ self.assertEqual('*',
+ resp.headers['access-control-allow-origin'])
+
+ # test allow_origin empty
+ container_cors = {'allow_origin': ''}
+ resp = self._get_CORS_response(
+ container_cors=container_cors, strict_mode=False)
+ self.assertEqual(200, resp.status_int)
+ self.assertEqual('http://foo.bar',
+ resp.headers['access-control-allow-origin'])
+
+ def test_CORS_valid_strict(self):
+ # test expose_headers to non-allowed origins
+ container_cors = {'allow_origin': 'http://not.foo.bar',
+ 'expose_headers': 'X-Object-Meta-Color '
+ 'X-Object-Meta-Color-Ex'}
+ resp = self._get_CORS_response(
+ container_cors=container_cors, strict_mode=True)
+
+ self.assertEqual(200, resp.status_int)
+ self.assertNotIn('access-control-expose-headers', resp.headers)
+ self.assertNotIn('access-control-allow-origin', resp.headers)
+
+ # test allow_origin *
+ container_cors = {'allow_origin': '*'}
+
+ resp = self._get_CORS_response(
+ container_cors=container_cors, strict_mode=True)
+ self.assertEqual(200, resp.status_int)
+ self.assertEqual('*',
+ resp.headers['access-control-allow-origin'])
+ self.assertEqual('red', resp.headers['x-object-meta-color'])
+ # X-Super-Secret is in the response, but not "exposed"
+ self.assertEqual('hush', resp.headers['x-super-secret'])
+ self.assertIn('access-control-expose-headers', resp.headers)
+ exposed = set(
+ h.strip() for h in
+ resp.headers['access-control-expose-headers'].split(','))
+ expected_exposed = set([
+ 'cache-control', 'content-language', 'content-type', 'expires',
+ 'last-modified', 'pragma', 'etag', 'x-timestamp', 'x-trans-id',
+ 'x-openstack-request-id', 'x-object-meta-color'])
+ self.assertEqual(expected_exposed, exposed)
+
+ # test allow_origin empty
+ container_cors = {'allow_origin': ''}
+ resp = self._get_CORS_response(
+ container_cors=container_cors, strict_mode=True)
+ self.assertNotIn('access-control-expose-headers', resp.headers)
+ self.assertNotIn('access-control-allow-origin', resp.headers)
+
+ def test_CORS_valid_with_obj_headers(self):
+ container_cors = {'allow_origin': 'http://foo.bar'}
+
+ def objectGET(controller, req):
+ return Response(headers={
+ 'X-Object-Meta-Color': 'red',
+ 'X-Super-Secret': 'hush',
+ 'Access-Control-Allow-Origin': 'http://obj.origin',
+ 'Access-Control-Expose-Headers': 'x-trans-id'
+ })
+
+ resp = self._get_CORS_response(
+ container_cors=container_cors, strict_mode=True,
+ object_get=objectGET)
+
+ self.assertEqual(200, resp.status_int)
+ self.assertEqual('http://obj.origin',
+ resp.headers['access-control-allow-origin'])
+ self.assertEqual('x-trans-id',
+ resp.headers['access-control-expose-headers'])
+
+ def _gather_x_container_headers(self, controller_call, req, *connect_args,
+ **kwargs):
+ header_list = kwargs.pop('header_list', ['X-Container-Device',
+ 'X-Container-Host',
+ 'X-Container-Partition'])
+ seen_headers = []
+
+ def capture_headers(ipaddr, port, device, partition, method,
+ path, headers=None, query_string=None):
+ captured = {}
+ for header in header_list:
+ captured[header] = headers.get(header)
+ seen_headers.append(captured)
+
+ with save_globals():
+ self.app.allow_account_management = True
+
+ set_http_connect(*connect_args, give_connect=capture_headers,
+ **kwargs)
+ resp = controller_call(req)
+ self.assertEqual(2, resp.status_int // 100) # sanity check
+
+ # don't care about the account/container HEADs, so chuck
+ # the first two requests
+ return sorted(seen_headers[2:],
+ key=lambda d: d.get(header_list[0]) or 'z')
+
+ def test_PUT_x_container_headers_with_equal_replicas(self):
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+ headers={'Content-Length': '5'}, body='12345')
+ controller = ReplicatedObjectController(
+ self.app, 'a', 'c', 'o')
+ seen_headers = self._gather_x_container_headers(
+ controller.PUT, req,
+ 200, 200, 201, 201, 201) # HEAD HEAD PUT PUT PUT
+ self.assertEqual(
+ seen_headers, [
+ {'X-Container-Host': '10.0.0.0:1000',
+ 'X-Container-Partition': '0',
+ 'X-Container-Device': 'sda'},
+ {'X-Container-Host': '10.0.0.1:1001',
+ 'X-Container-Partition': '0',
+ 'X-Container-Device': 'sdb'},
+ {'X-Container-Host': '10.0.0.2:1002',
+ 'X-Container-Partition': '0',
+ 'X-Container-Device': 'sdc'}])
+
+ def test_PUT_x_container_headers_with_fewer_container_replicas(self):
+ self.app.container_ring.set_replicas(2)
+
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+ headers={'Content-Length': '5'}, body='12345')
+ controller = ReplicatedObjectController(
+ self.app, 'a', 'c', 'o')
+ seen_headers = self._gather_x_container_headers(
+ controller.PUT, req,
+ 200, 200, 201, 201, 201) # HEAD HEAD PUT PUT PUT
+
+ self.assertEqual(
+ seen_headers, [
+ {'X-Container-Host': '10.0.0.0:1000',
+ 'X-Container-Partition': '0',
+ 'X-Container-Device': 'sda'},
+ {'X-Container-Host': '10.0.0.0:1000',
+ 'X-Container-Partition': '0',
+ 'X-Container-Device': 'sda'},
+ {'X-Container-Host': '10.0.0.1:1001',
+ 'X-Container-Partition': '0',
+ 'X-Container-Device': 'sdb'}])
+
+ def test_PUT_x_container_headers_with_more_container_replicas(self):
+ self.app.container_ring.set_replicas(4)
+
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+ headers={'Content-Length': '5'}, body='12345')
+ controller = ReplicatedObjectController(
+ self.app, 'a', 'c', 'o')
+ seen_headers = self._gather_x_container_headers(
+ controller.PUT, req,
+ 200, 200, 201, 201, 201) # HEAD HEAD PUT PUT PUT
+
+ self.assertEqual(
+ seen_headers, [
+ {'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003',
+ 'X-Container-Partition': '0',
+ 'X-Container-Device': 'sda,sdd'},
+ {'X-Container-Host': '10.0.0.1:1001',
+ 'X-Container-Partition': '0',
+ 'X-Container-Device': 'sdb'},
+ {'X-Container-Host': '10.0.0.2:1002',
+ 'X-Container-Partition': '0',
+ 'X-Container-Device': 'sdc'}])
+
+ def test_POST_x_container_headers_with_more_container_replicas(self):
+ self.app.container_ring.set_replicas(4)
+ self.app.object_post_as_copy = False
+
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'POST'},
+ headers={'Content-Type': 'application/stuff'})
+ controller = ReplicatedObjectController(
+ self.app, 'a', 'c', 'o')
+ seen_headers = self._gather_x_container_headers(
+ controller.POST, req,
+ 200, 200, 200, 200, 200) # HEAD HEAD POST POST POST
+
+ self.assertEqual(
+ seen_headers, [
+ {'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003',
+ 'X-Container-Partition': '0',
+ 'X-Container-Device': 'sda,sdd'},
+ {'X-Container-Host': '10.0.0.1:1001',
+ 'X-Container-Partition': '0',
+ 'X-Container-Device': 'sdb'},
+ {'X-Container-Host': '10.0.0.2:1002',
+ 'X-Container-Partition': '0',
+ 'X-Container-Device': 'sdc'}])
+
+ def test_DELETE_x_container_headers_with_more_container_replicas(self):
+ self.app.container_ring.set_replicas(4)
+
+ req = Request.blank('/v1/a/c/o',
+ environ={'REQUEST_METHOD': 'DELETE'},
+ headers={'Content-Type': 'application/stuff'})
+ controller = ReplicatedObjectController(
+ self.app, 'a', 'c', 'o')
+ seen_headers = self._gather_x_container_headers(
+ controller.DELETE, req,
+ 200, 200, 200, 200, 200) # HEAD HEAD DELETE DELETE DELETE
+
+ self.assertEqual(seen_headers, [
+ {'X-Container-Host': '10.0.0.0:1000,10.0.0.3:1003',
+ 'X-Container-Partition': '0',
+ 'X-Container-Device': 'sda,sdd'},
+ {'X-Container-Host': '10.0.0.1:1001',
+ 'X-Container-Partition': '0',
+ 'X-Container-Device': 'sdb'},
+ {'X-Container-Host': '10.0.0.2:1002',
+ 'X-Container-Partition': '0',
+ 'X-Container-Device': 'sdc'}
+ ])
+
+ @mock.patch('time.time', new=lambda: STATIC_TIME)
+ def test_PUT_x_delete_at_with_fewer_container_replicas(self):
+ self.app.container_ring.set_replicas(2)
+
+ delete_at_timestamp = int(time.time()) + 100000
+ delete_at_container = utils.get_expirer_container(
+ delete_at_timestamp, self.app.expiring_objects_container_divisor,
+ 'a', 'c', 'o')
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+ headers={'Content-Type': 'application/stuff',
+ 'Content-Length': '0',
+ 'X-Delete-At': str(delete_at_timestamp)})
+ controller = ReplicatedObjectController(
+ self.app, 'a', 'c', 'o')
+ seen_headers = self._gather_x_container_headers(
+ controller.PUT, req,
+ 200, 200, 201, 201, 201, # HEAD HEAD PUT PUT PUT
+ header_list=('X-Delete-At-Host', 'X-Delete-At-Device',
+ 'X-Delete-At-Partition', 'X-Delete-At-Container'))
+
+ self.assertEqual(seen_headers, [
+ {'X-Delete-At-Host': '10.0.0.0:1000',
+ 'X-Delete-At-Container': delete_at_container,
+ 'X-Delete-At-Partition': '0',
+ 'X-Delete-At-Device': 'sda'},
+ {'X-Delete-At-Host': '10.0.0.1:1001',
+ 'X-Delete-At-Container': delete_at_container,
+ 'X-Delete-At-Partition': '0',
+ 'X-Delete-At-Device': 'sdb'},
+ {'X-Delete-At-Host': None,
+ 'X-Delete-At-Container': None,
+ 'X-Delete-At-Partition': None,
+ 'X-Delete-At-Device': None}
+ ])
+
+ @mock.patch('time.time', new=lambda: STATIC_TIME)
+ def test_PUT_x_delete_at_with_more_container_replicas(self):
+ self.app.container_ring.set_replicas(4)
+ self.app.expiring_objects_account = 'expires'
+ self.app.expiring_objects_container_divisor = 60
+
+ delete_at_timestamp = int(time.time()) + 100000
+ delete_at_container = utils.get_expirer_container(
+ delete_at_timestamp, self.app.expiring_objects_container_divisor,
+ 'a', 'c', 'o')
+ req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
+ headers={'Content-Type': 'application/stuff',
+ 'Content-Length': 0,
+ 'X-Delete-At': str(delete_at_timestamp)})
+ controller = ReplicatedObjectController(
+ self.app, 'a', 'c', 'o')
+ seen_headers = self._gather_x_container_headers(
+ controller.PUT, req,
+ 200, 200, 201, 201, 201, # HEAD HEAD PUT PUT PUT
+ header_list=('X-Delete-At-Host', 'X-Delete-At-Device',
+ 'X-Delete-At-Partition', 'X-Delete-At-Container'))
+ self.assertEqual(seen_headers, [
+ {'X-Delete-At-Host': '10.0.0.0:1000,10.0.0.3:1003',
+ 'X-Delete-At-Container': delete_at_container,
+ 'X-Delete-At-Partition': '0',
+ 'X-Delete-At-Device': 'sda,sdd'},
+ {'X-Delete-At-Host': '10.0.0.1:1001',
+ 'X-Delete-At-Container': delete_at_container,
+ 'X-Delete-At-Partition': '0',
+ 'X-Delete-At-Device': 'sdb'},
+ {'X-Delete-At-Host': '10.0.0.2:1002',
+ 'X-Delete-At-Container': delete_at_container,
+ 'X-Delete-At-Partition': '0',
+ 'X-Delete-At-Device': 'sdc'}
+ ])
+
+
if __name__ == '__main__':
+ setup()
unittest.main()
diff --git a/test/unit/proxy/test_mem_server.py b/test/unit/proxy/test_mem_server.py
index 2221ee926eaa..b0dff95ddbce 100644
--- a/test/unit/proxy/test_mem_server.py
+++ b/test/unit/proxy/test_mem_server.py
@@ -17,6 +17,8 @@ import unittest
from test.unit.proxy import test_server
from test.unit.proxy.test_server import teardown
+from test.unit.proxy.controllers import test_account
+from test.unit.proxy.controllers import test_container
from swift.obj import mem_server
@@ -55,16 +57,16 @@ class TestObjectController(test_server.TestObjectController):
pass
-class TestContainerController(test_server.TestContainerController):
+class TestContainerController(test_container.TestContainerController):
pass
-class TestAccountController(test_server.TestAccountController):
+class TestAccountController(test_account.TestAccountController):
pass
class TestAccountControllerFakeGetResponse(
- test_server.TestAccountControllerFakeGetResponse):
+ test_account.TestAccountControllerFakeGetResponse):
pass
diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py
index 08e9e2abd3e5..89560416bf0a 100644
--- a/test/unit/proxy/test_server.py
+++ b/test/unit/proxy/test_server.py
@@ -37,24 +37,23 @@ import operator
import functools
from swift.obj import diskfile
import re
-import random
from collections import defaultdict
import uuid
import mock
from eventlet import sleep, spawn, wsgi, listen, Timeout, debug
from eventlet.green import httplib
-from six import BytesIO
from six import StringIO
from six.moves import range
from six.moves.urllib.parse import quote
-from swift.common.utils import hash_path, storage_directory, \
- parse_content_type, parse_mime_headers, \
- iter_multipart_mime_documents, public
+from swift.common.utils import (
+ hash_path, storage_directory,
+ parse_content_type, parse_mime_headers, iter_multipart_mime_documents,
+ public)
from test.unit import (
- connect_tcp, readuntil2crlfs, FakeLogger, fake_http_connect, FakeRing,
+ connect_tcp, readuntil2crlfs, FakeLogger, FakeRing,
FakeMemcache, debug_logger, patch_policies, write_fake_ring,
mocked_http_conn, DEFAULT_TEST_EC_TYPE, make_timestamp_iter)
from swift.proxy import server as proxy_server
@@ -62,7 +61,6 @@ from swift.proxy.controllers.obj import ReplicatedObjectController
from swift.obj import server as object_server
from swift.common.middleware import proxy_logging, versioned_writes, \
copy
-from swift.common.middleware.acl import parse_acl, format_acl
from swift.common.exceptions import ChunkReadTimeout, DiskFileNotExist, \
APIVersionError, ChunkWriteTimeout
from swift.common import utils, constraints
@@ -78,9 +76,11 @@ from swift.common.swob import Request, Response, HTTPUnauthorized, \
HTTPException, HTTPBadRequest
from swift.common.storage_policy import StoragePolicy, POLICIES
import swift.common.request_helpers
-from swift.common.request_helpers import get_sys_meta_prefix
from test.unit.helpers import setup_servers, teardown_servers
+from test.unit.proxy import (
+ save_globals, set_http_connect, FakeMemcacheReturnsNone)
+
# mocks
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
@@ -168,54 +168,6 @@ def set_node_errors(proxy_app, ring_node, value, last_error):
stats['last_error'] = last_error
-class FakeMemcacheReturnsNone(FakeMemcache):
-
- def get(self, key):
- # Returns None as the timestamp of the container; assumes we're only
- # using the FakeMemcache for container existence checks.
- return None
-
-
-@contextmanager
-def save_globals():
- orig_http_connect = getattr(swift.proxy.controllers.base, 'http_connect',
- None)
- orig_account_info = getattr(swift.proxy.controllers.Controller,
- 'account_info', None)
- orig_container_info = getattr(swift.proxy.controllers.Controller,
- 'container_info', None)
-
- try:
- yield True
- finally:
- swift.proxy.controllers.Controller.account_info = orig_account_info
- swift.proxy.controllers.base.http_connect = orig_http_connect
- swift.proxy.controllers.obj.http_connect = orig_http_connect
- swift.proxy.controllers.account.http_connect = orig_http_connect
- swift.proxy.controllers.container.http_connect = orig_http_connect
- swift.proxy.controllers.Controller.container_info = orig_container_info
-
-
-def set_http_connect(*args, **kwargs):
- new_connect = fake_http_connect(*args, **kwargs)
- swift.proxy.controllers.base.http_connect = new_connect
- swift.proxy.controllers.obj.http_connect = new_connect
- swift.proxy.controllers.account.http_connect = new_connect
- swift.proxy.controllers.container.http_connect = new_connect
- return new_connect
-
-
-def _make_callback_func(calls):
- def callback(ipaddr, port, device, partition, method, path,
- headers=None, query_string=None, ssl=False):
- context = {}
- context['method'] = method
- context['path'] = path
- context['headers'] = headers or {}
- calls.append(context)
- return callback
-
-
def _limit_max_file_size(f):
"""
This will limit constraints.MAX_FILE_SIZE for the duration of the
@@ -1217,174 +1169,6 @@ class TestObjectController(unittest.TestCase):
break
@unpatch_policies
- def test_policy_IO(self):
- def check_file(policy, cont, devs, check_val):
- partition, nodes = policy.object_ring.get_nodes('a', cont, 'o')
- conf = {'devices': _testdir, 'mount_check': 'false'}
- df_mgr = diskfile.DiskFileManager(conf, FakeLogger())
- for dev in devs:
- file = df_mgr.get_diskfile(dev, partition, 'a',
- cont, 'o',
- policy=policy)
- if check_val is True:
- file.open()
-
- prolis = _test_sockets[0]
- prosrv = _test_servers[0]
-
- # check policy 0: put file on c, read it back, check loc on disk
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- obj = 'test_object0'
- path = '/v1/a/c/o'
- fd.write('PUT %s HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Storage-Token: t\r\n'
- 'Content-Length: %s\r\n'
- 'Content-Type: text/plain\r\n'
- '\r\n%s' % (path, str(len(obj)), obj))
- fd.flush()
- headers = readuntil2crlfs(fd)
- exp = 'HTTP/1.1 201'
- self.assertEqual(headers[:len(exp)], exp)
- req = Request.blank(path,
- environ={'REQUEST_METHOD': 'GET'},
- headers={'Content-Type':
- 'text/plain'})
- res = req.get_response(prosrv)
- self.assertEqual(res.status_int, 200)
- self.assertEqual(res.body, obj)
-
- check_file(POLICIES[0], 'c', ['sda1', 'sdb1'], True)
- check_file(POLICIES[0], 'c', ['sdc1', 'sdd1', 'sde1', 'sdf1'], False)
-
- # check policy 1: put file on c1, read it back, check loc on disk
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- path = '/v1/a/c1/o'
- obj = 'test_object1'
- fd.write('PUT %s HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Storage-Token: t\r\n'
- 'Content-Length: %s\r\n'
- 'Content-Type: text/plain\r\n'
- '\r\n%s' % (path, str(len(obj)), obj))
- fd.flush()
- headers = readuntil2crlfs(fd)
- self.assertEqual(headers[:len(exp)], exp)
- req = Request.blank(path,
- environ={'REQUEST_METHOD': 'GET'},
- headers={'Content-Type':
- 'text/plain'})
- res = req.get_response(prosrv)
- self.assertEqual(res.status_int, 200)
- self.assertEqual(res.body, obj)
-
- check_file(POLICIES[1], 'c1', ['sdc1', 'sdd1'], True)
- check_file(POLICIES[1], 'c1', ['sda1', 'sdb1', 'sde1', 'sdf1'], False)
-
- # check policy 2: put file on c2, read it back, check loc on disk
- sock = connect_tcp(('localhost', prolis.getsockname()[1]))
- fd = sock.makefile()
- path = '/v1/a/c2/o'
- obj = 'test_object2'
- fd.write('PUT %s HTTP/1.1\r\n'
- 'Host: localhost\r\n'
- 'Connection: close\r\n'
- 'X-Storage-Token: t\r\n'
- 'Content-Length: %s\r\n'
- 'Content-Type: text/plain\r\n'
- '\r\n%s' % (path, str(len(obj)), obj))
- fd.flush()
- headers = readuntil2crlfs(fd)
- self.assertEqual(headers[:len(exp)], exp)
- req = Request.blank(path,
- environ={'REQUEST_METHOD': 'GET'},
- headers={'Content-Type':
- 'text/plain'})
- res = req.get_response(prosrv)
- self.assertEqual(res.status_int, 200)
- self.assertEqual(res.body, obj)
-
- check_file(POLICIES[2], 'c2', ['sde1', 'sdf1'], True)
- check_file(POLICIES[2], 'c2', ['sda1', 'sdb1', 'sdc1', 'sdd1'], False)
-
- @unpatch_policies
- def test_policy_IO_override(self):
- if hasattr(_test_servers[-1], '_filesystem'):
- # ironically, the _filesystem attribute on the object server means
- # the in-memory diskfile is in use, so this test does not apply
- return
-
- prosrv = _test_servers[0]
-
- # validate container policy is 1
- req = Request.blank('/v1/a/c1', method='HEAD')
- res = req.get_response(prosrv)
- self.assertEqual(res.status_int, 204) # sanity check
- self.assertEqual(POLICIES[1].name, res.headers['x-storage-policy'])
-
- # check overrides: put it in policy 2 (not where the container says)
- req = Request.blank(
- '/v1/a/c1/wrong-o',
- environ={'REQUEST_METHOD': 'PUT',
- 'wsgi.input': BytesIO(b"hello")},
- headers={'Content-Type': 'text/plain',
- 'Content-Length': '5',
- 'X-Backend-Storage-Policy-Index': '2'})
- res = req.get_response(prosrv)
- self.assertEqual(res.status_int, 201) # sanity check
-
- # go to disk to make sure it's there
- partition, nodes = prosrv.get_object_ring(2).get_nodes(
- 'a', 'c1', 'wrong-o')
- node = nodes[0]
- conf = {'devices': _testdir, 'mount_check': 'false'}
- df_mgr = diskfile.DiskFileManager(conf, FakeLogger())
- df = df_mgr.get_diskfile(node['device'], partition, 'a',
- 'c1', 'wrong-o', policy=POLICIES[2])
- with df.open():
- contents = ''.join(df.reader())
- self.assertEqual(contents, "hello")
-
- # can't get it from the normal place
- req = Request.blank('/v1/a/c1/wrong-o',
- environ={'REQUEST_METHOD': 'GET'},
- headers={'Content-Type': 'text/plain'})
- res = req.get_response(prosrv)
- self.assertEqual(res.status_int, 404) # sanity check
-
- # but we can get it from policy 2
- req = Request.blank('/v1/a/c1/wrong-o',
- environ={'REQUEST_METHOD': 'GET'},
- headers={'Content-Type': 'text/plain',
- 'X-Backend-Storage-Policy-Index': '2'})
-
- res = req.get_response(prosrv)
- self.assertEqual(res.status_int, 200)
- self.assertEqual(res.body, 'hello')
-
- # and we can delete it the same way
- req = Request.blank('/v1/a/c1/wrong-o',
- environ={'REQUEST_METHOD': 'DELETE'},
- headers={'Content-Type': 'text/plain',
- 'X-Backend-Storage-Policy-Index': '2'})
-
- res = req.get_response(prosrv)
- self.assertEqual(res.status_int, 204)
-
- df = df_mgr.get_diskfile(node['device'], partition, 'a',
- 'c1', 'wrong-o', policy=POLICIES[2])
- try:
- df.open()
- except DiskFileNotExist as e:
- self.assertGreater(float(e.timestamp), 0)
- else:
- self.fail('did not raise DiskFileNotExist')
-
- @unpatch_policies
def test_GET_newest_large_file(self):
prolis = _test_sockets[0]
prosrv = _test_servers[0]
@@ -6460,1960 +6244,6 @@ class TestObjectECRangedGET(unittest.TestCase):
self.assertEqual(second_byterange.get_payload(), self.obj[4090:5011])
-@patch_policies([
- StoragePolicy(0, 'zero', True, object_ring=FakeRing(base_port=3000)),
- StoragePolicy(1, 'one', False, object_ring=FakeRing(base_port=3000)),
- StoragePolicy(2, 'two', False, True, object_ring=FakeRing(base_port=3000))
-])
-class TestContainerController(unittest.TestCase):
- "Test swift.proxy_server.ContainerController"
-
- def setUp(self):
- self.app = proxy_server.Application(
- None, FakeMemcache(),
- account_ring=FakeRing(),
- container_ring=FakeRing(base_port=2000),
- logger=debug_logger())
-
- def test_convert_policy_to_index(self):
- controller = swift.proxy.controllers.ContainerController(self.app,
- 'a', 'c')
- expected = {
- 'zero': 0,
- 'ZeRo': 0,
- 'one': 1,
- 'OnE': 1,
- }
- for name, index in expected.items():
- req = Request.blank('/a/c', headers={'Content-Length': '0',
- 'Content-Type': 'text/plain',
- 'X-Storage-Policy': name})
- self.assertEqual(controller._convert_policy_to_index(req), index)
- # default test
- req = Request.blank('/a/c', headers={'Content-Length': '0',
- 'Content-Type': 'text/plain'})
- self.assertIsNone(controller._convert_policy_to_index(req))
- # negative test
- req = Request.blank('/a/c',
- headers={'Content-Length': '0',
- 'Content-Type': 'text/plain',
- 'X-Storage-Policy': 'nada'})
- self.assertRaises(HTTPException, controller._convert_policy_to_index,
- req)
- # storage policy two is deprecated
- req = Request.blank('/a/c', headers={'Content-Length': '0',
- 'Content-Type': 'text/plain',
- 'X-Storage-Policy': 'two'})
- self.assertRaises(HTTPException, controller._convert_policy_to_index,
- req)
-
- def test_convert_index_to_name(self):
- policy = random.choice(list(POLICIES))
- req = Request.blank('/v1/a/c')
- with mocked_http_conn(
- 200, 200,
- headers={'X-Backend-Storage-Policy-Index': int(policy)},
- ) as fake_conn:
- resp = req.get_response(self.app)
- self.assertRaises(StopIteration, fake_conn.code_iter.next)
- self.assertEqual(resp.status_int, 200)
- self.assertEqual(resp.headers['X-Storage-Policy'], policy.name)
-
- def test_no_convert_index_to_name_when_container_not_found(self):
- policy = random.choice(list(POLICIES))
- req = Request.blank('/v1/a/c')
- with mocked_http_conn(
- 200, 404, 404, 404,
- headers={'X-Backend-Storage-Policy-Index':
- int(policy)}) as fake_conn:
- resp = req.get_response(self.app)
- self.assertRaises(StopIteration, fake_conn.code_iter.next)
- self.assertEqual(resp.status_int, 404)
- self.assertIsNone(resp.headers['X-Storage-Policy'])
-
- def test_error_convert_index_to_name(self):
- req = Request.blank('/v1/a/c')
- with mocked_http_conn(
- 200, 200,
- headers={'X-Backend-Storage-Policy-Index': '-1'}) as fake_conn:
- resp = req.get_response(self.app)
- self.assertRaises(StopIteration, fake_conn.code_iter.next)
- self.assertEqual(resp.status_int, 200)
- self.assertIsNone(resp.headers['X-Storage-Policy'])
- error_lines = self.app.logger.get_lines_for_level('error')
- self.assertEqual(2, len(error_lines))
- for msg in error_lines:
- expected = "Could not translate " \
- "X-Backend-Storage-Policy-Index ('-1')"
- self.assertIn(expected, msg)
-
- def test_transfer_headers(self):
- src_headers = {'x-remove-versions-location': 'x',
- 'x-container-read': '*:user',
- 'x-remove-container-sync-key': 'x'}
- dst_headers = {'x-versions-location': 'backup'}
- controller = swift.proxy.controllers.ContainerController(self.app,
- 'a', 'c')
- controller.transfer_headers(src_headers, dst_headers)
- expected_headers = {'x-versions-location': '',
- 'x-container-read': '*:user',
- 'x-container-sync-key': ''}
- self.assertEqual(dst_headers, expected_headers)
-
- def assert_status_map(self, method, statuses, expected,
- raise_exc=False, missing_container=False):
- with save_globals():
- kwargs = {}
- if raise_exc:
- kwargs['raise_exc'] = raise_exc
- kwargs['missing_container'] = missing_container
- set_http_connect(*statuses, **kwargs)
- self.app.memcache.store = {}
- req = Request.blank('/v1/a/c', headers={'Content-Length': '0',
- 'Content-Type': 'text/plain'})
- self.app.update_request(req)
- res = method(req)
- self.assertEqual(res.status_int, expected)
- set_http_connect(*statuses, **kwargs)
- self.app.memcache.store = {}
- req = Request.blank('/v1/a/c/', headers={'Content-Length': '0',
- 'Content-Type': 'text/plain'})
- self.app.update_request(req)
- res = method(req)
- self.assertEqual(res.status_int, expected)
-
- def test_HEAD_GET(self):
- with save_globals():
- controller = proxy_server.ContainerController(self.app, 'a', 'c')
-
- def test_status_map(statuses, expected,
- c_expected=None, a_expected=None, **kwargs):
- set_http_connect(*statuses, **kwargs)
- self.app.memcache.store = {}
- req = Request.blank('/v1/a/c', {})
- self.app.update_request(req)
- res = controller.HEAD(req)
- self.assertEqual(res.status[:len(str(expected))],
- str(expected))
- infocache = res.environ.get('swift.infocache', {})
- if expected < 400:
- self.assertIn('x-works', res.headers)
- self.assertEqual(res.headers['x-works'], 'yes')
- if expected < 300:
- self.assertIn('last-modified', res.headers)
- self.assertEqual(res.headers['last-modified'], '1')
- if c_expected:
- self.assertIn('container/a/c', infocache)
- self.assertEqual(
- infocache['container/a/c']['status'],
- c_expected)
- else:
- self.assertNotIn('container/a/c', infocache)
- if a_expected:
- self.assertIn('account/a', infocache)
- self.assertEqual(infocache['account/a']['status'],
- a_expected)
- else:
- self.assertNotIn('account/a', res.environ)
-
- set_http_connect(*statuses, **kwargs)
- self.app.memcache.store = {}
- req = Request.blank('/v1/a/c', {})
- self.app.update_request(req)
- res = controller.GET(req)
- self.assertEqual(res.status[:len(str(expected))],
- str(expected))
- infocache = res.environ.get('swift.infocache', {})
- if expected < 400:
- self.assertIn('x-works', res.headers)
- self.assertEqual(res.headers['x-works'], 'yes')
- if expected < 300:
- self.assertIn('last-modified', res.headers)
- self.assertEqual(res.headers['last-modified'], '1')
- if c_expected:
- self.assertIn('container/a/c', infocache)
- self.assertEqual(
- infocache['container/a/c']['status'],
- c_expected)
- else:
- self.assertNotIn('container/a/c', infocache)
- if a_expected:
- self.assertIn('account/a', infocache)
- self.assertEqual(infocache['account/a']['status'],
- a_expected)
- else:
- self.assertNotIn('account/a', infocache)
- # In all the following tests cache 200 for account
- # return and cache vary for container
- # return 200 and cache 200 for account and container
- test_status_map((200, 200, 404, 404), 200, 200, 200)
- test_status_map((200, 200, 500, 404), 200, 200, 200)
- # return 304 don't cache container
- test_status_map((200, 304, 500, 404), 304, None, 200)
- # return 404 and cache 404 for container
- test_status_map((200, 404, 404, 404), 404, 404, 200)
- test_status_map((200, 404, 404, 500), 404, 404, 200)
- # return 503, don't cache container
- test_status_map((200, 500, 500, 500), 503, None, 200)
- self.assertFalse(self.app.account_autocreate)
-
- # return 404 (as account is not found) and don't cache container
- test_status_map((404, 404, 404), 404, None, 404)
-
- # cache a 204 for the account because it's sort of like it
- # exists
- self.app.account_autocreate = True
- test_status_map((404, 404, 404), 404, None, 204)
-
- def test_PUT_policy_headers(self):
- backend_requests = []
-
- def capture_requests(ipaddr, port, device, partition, method,
- path, headers=None, query_string=None):
- if method == 'PUT':
- backend_requests.append(headers)
-
- def test_policy(requested_policy):
- with save_globals():
- mock_conn = set_http_connect(200, 201, 201, 201,
- give_connect=capture_requests)
- self.app.memcache.store = {}
- req = Request.blank('/v1/a/test', method='PUT',
- headers={'Content-Length': 0})
- if requested_policy:
- expected_policy = requested_policy
- req.headers['X-Storage-Policy'] = policy.name
- else:
- expected_policy = POLICIES.default
- res = req.get_response(self.app)
- if expected_policy.is_deprecated:
- self.assertEqual(res.status_int, 400)
- self.assertEqual(0, len(backend_requests))
- expected = 'is deprecated'
- self.assertIn(expected, res.body,
- '%r did not include %r' % (
- res.body, expected))
- return
- self.assertEqual(res.status_int, 201)
- self.assertEqual(
- expected_policy.object_ring.replicas,
- len(backend_requests))
- for headers in backend_requests:
- if not requested_policy:
- self.assertNotIn('X-Backend-Storage-Policy-Index',
- headers)
- self.assertIn('X-Backend-Storage-Policy-Default',
- headers)
- self.assertEqual(
- int(expected_policy),
- int(headers['X-Backend-Storage-Policy-Default']))
- else:
- self.assertIn('X-Backend-Storage-Policy-Index',
- headers)
- self.assertEqual(int(headers
- ['X-Backend-Storage-Policy-Index']),
- int(policy))
- # make sure all mocked responses are consumed
- self.assertRaises(StopIteration, mock_conn.code_iter.next)
-
- test_policy(None) # no policy header
- for policy in POLICIES:
- backend_requests = [] # reset backend requests
- test_policy(policy)
-
- def test_PUT(self):
- with save_globals():
- controller = proxy_server.ContainerController(self.app, 'account',
- 'container')
-
- def test_status_map(statuses, expected, **kwargs):
- set_http_connect(*statuses, **kwargs)
- self.app.memcache.store = {}
- req = Request.blank('/v1/a/c', {})
- req.content_length = 0
- self.app.update_request(req)
- res = controller.PUT(req)
- expected = str(expected)
- self.assertEqual(res.status[:len(expected)], expected)
-
- test_status_map((200, 201, 201, 201), 201, missing_container=True)
- test_status_map((200, 201, 201, 500), 201, missing_container=True)
- test_status_map((200, 204, 404, 404), 404, missing_container=True)
- test_status_map((200, 204, 500, 404), 503, missing_container=True)
- self.assertFalse(self.app.account_autocreate)
- test_status_map((404, 404, 404), 404, missing_container=True)
- self.app.account_autocreate = True
- # fail to retrieve account info
- test_status_map(
- (503, 503, 503), # account_info fails on 503
- 404, missing_container=True)
- # account fail after creation
- test_status_map(
- (404, 404, 404, # account_info fails on 404
- 201, 201, 201, # PUT account
- 404, 404, 404), # account_info fail
- 404, missing_container=True)
- test_status_map(
- (503, 503, 404, # account_info fails on 404
- 503, 503, 503, # PUT account
- 503, 503, 404), # account_info fail
- 404, missing_container=True)
- # put fails
- test_status_map(
- (404, 404, 404, # account_info fails on 404
- 201, 201, 201, # PUT account
- 200, # account_info success
- 503, 503, 201), # put container fail
- 503, missing_container=True)
- # all goes according to plan
- test_status_map(
- (404, 404, 404, # account_info fails on 404
- 201, 201, 201, # PUT account
- 200, # account_info success
- 201, 201, 201), # put container success
- 201, missing_container=True)
- test_status_map(
- (503, 404, 404, # account_info fails on 404
- 503, 201, 201, # PUT account
- 503, 200, # account_info success
- 503, 201, 201), # put container success
- 201, missing_container=True)
-
- def test_PUT_autocreate_account_with_sysmeta(self):
- # x-account-sysmeta headers in a container PUT request should be
- # transferred to the account autocreate PUT request
- with save_globals():
- controller = proxy_server.ContainerController(self.app, 'account',
- 'container')
-
- def test_status_map(statuses, expected, headers=None, **kwargs):
- set_http_connect(*statuses, **kwargs)
- self.app.memcache.store = {}
- req = Request.blank('/v1/a/c', {}, headers=headers)
- req.content_length = 0
- self.app.update_request(req)
- res = controller.PUT(req)
- expected = str(expected)
- self.assertEqual(res.status[:len(expected)], expected)
-
- self.app.account_autocreate = True
- calls = []
- callback = _make_callback_func(calls)
- key, value = 'X-Account-Sysmeta-Blah', 'something'
- headers = {key: value}
-
- # all goes according to plan
- test_status_map(
- (404, 404, 404, # account_info fails on 404
- 201, 201, 201, # PUT account
- 200, # account_info success
- 201, 201, 201), # put container success
- 201, missing_container=True,
- headers=headers,
- give_connect=callback)
-
- self.assertEqual(10, len(calls))
- for call in calls[3:6]:
- self.assertEqual('/account', call['path'])
- self.assertIn(key, call['headers'],
- '%s call, key %s missing in headers %s' % (
- call['method'], key, call['headers']))
- self.assertEqual(value, call['headers'][key])
-
- def test_POST(self):
- with save_globals():
- controller = proxy_server.ContainerController(self.app, 'account',
- 'container')
-
- def test_status_map(statuses, expected, **kwargs):
- set_http_connect(*statuses, **kwargs)
- self.app.memcache.store = {}
- req = Request.blank('/v1/a/c', {})
- req.content_length = 0
- self.app.update_request(req)
- res = controller.POST(req)
- expected = str(expected)
- self.assertEqual(res.status[:len(expected)], expected)
-
- test_status_map((200, 201, 201, 201), 201, missing_container=True)
- test_status_map((200, 201, 201, 500), 201, missing_container=True)
- test_status_map((200, 204, 404, 404), 404, missing_container=True)
- test_status_map((200, 204, 500, 404), 503, missing_container=True)
- self.assertFalse(self.app.account_autocreate)
- test_status_map((404, 404, 404), 404, missing_container=True)
- self.app.account_autocreate = True
- test_status_map((404, 404, 404), 404, missing_container=True)
-
- def test_PUT_max_containers_per_account(self):
- with save_globals():
- self.app.max_containers_per_account = 12346
- controller = proxy_server.ContainerController(self.app, 'account',
- 'container')
- self.assert_status_map(controller.PUT,
- (200, 201, 201, 201), 201,
- missing_container=True)
-
- self.app.max_containers_per_account = 12345
- controller = proxy_server.ContainerController(self.app, 'account',
- 'container')
- self.assert_status_map(controller.PUT,
- (200, 200, 201, 201, 201), 201,
- missing_container=True)
-
- controller = proxy_server.ContainerController(self.app, 'account',
- 'container_new')
-
- self.assert_status_map(controller.PUT, (200, 404, 404, 404), 403,
- missing_container=True)
-
- self.app.max_containers_per_account = 12345
- self.app.max_containers_whitelist = ['account']
- controller = proxy_server.ContainerController(self.app, 'account',
- 'container')
- self.assert_status_map(controller.PUT,
- (200, 201, 201, 201), 201,
- missing_container=True)
-
- def test_PUT_max_container_name_length(self):
- with save_globals():
- limit = constraints.MAX_CONTAINER_NAME_LENGTH
- controller = proxy_server.ContainerController(self.app, 'account',
- '1' * limit)
- self.assert_status_map(controller.PUT,
- (200, 201, 201, 201), 201,
- missing_container=True)
- controller = proxy_server.ContainerController(self.app, 'account',
- '2' * (limit + 1))
- self.assert_status_map(controller.PUT, (201, 201, 201), 400,
- missing_container=True)
-
- def test_PUT_connect_exceptions(self):
- with save_globals():
- controller = proxy_server.ContainerController(self.app, 'account',
- 'container')
- self.assert_status_map(controller.PUT, (200, 201, 201, -1), 201,
- missing_container=True)
- self.assert_status_map(controller.PUT, (200, 201, -1, -1), 503,
- missing_container=True)
- self.assert_status_map(controller.PUT, (200, 503, 503, -1), 503,
- missing_container=True)
-
- def test_acc_missing_returns_404(self):
- for meth in ('DELETE', 'PUT'):
- with save_globals():
- self.app.memcache = FakeMemcacheReturnsNone()
- self.app._error_limiting = {}
- controller = proxy_server.ContainerController(self.app,
- 'account',
- 'container')
- if meth == 'PUT':
- set_http_connect(200, 200, 200, 200, 200, 200,
- missing_container=True)
- else:
- set_http_connect(200, 200, 200, 200)
- self.app.memcache.store = {}
- req = Request.blank('/v1/a/c',
- environ={'REQUEST_METHOD': meth})
- self.app.update_request(req)
- resp = getattr(controller, meth)(req)
- self.assertEqual(resp.status_int, 200)
-
- set_http_connect(404, 404, 404, 200, 200, 200)
- # Make sure it is a blank request wthout env caching
- req = Request.blank('/v1/a/c',
- environ={'REQUEST_METHOD': meth})
- resp = getattr(controller, meth)(req)
- self.assertEqual(resp.status_int, 404)
-
- set_http_connect(503, 404, 404)
- # Make sure it is a blank request wthout env caching
- req = Request.blank('/v1/a/c',
- environ={'REQUEST_METHOD': meth})
- resp = getattr(controller, meth)(req)
- self.assertEqual(resp.status_int, 404)
-
- set_http_connect(503, 404, raise_exc=True)
- # Make sure it is a blank request wthout env caching
- req = Request.blank('/v1/a/c',
- environ={'REQUEST_METHOD': meth})
- resp = getattr(controller, meth)(req)
- self.assertEqual(resp.status_int, 404)
-
- for dev in self.app.account_ring.devs:
- set_node_errors(self.app, dev,
- self.app.error_suppression_limit + 1,
- time.time())
- set_http_connect(200, 200, 200, 200, 200, 200)
- # Make sure it is a blank request wthout env caching
- req = Request.blank('/v1/a/c',
- environ={'REQUEST_METHOD': meth})
- resp = getattr(controller, meth)(req)
- self.assertEqual(resp.status_int, 404)
-
- def test_put_locking(self):
-
- class MockMemcache(FakeMemcache):
-
- def __init__(self, allow_lock=None):
- self.allow_lock = allow_lock
- super(MockMemcache, self).__init__()
-
- @contextmanager
- def soft_lock(self, key, timeout=0, retries=5):
- if self.allow_lock:
- yield True
- else:
- raise NotImplementedError
-
- with save_globals():
- controller = proxy_server.ContainerController(self.app, 'account',
- 'container')
- self.app.memcache = MockMemcache(allow_lock=True)
- set_http_connect(200, 201, 201, 201,
- missing_container=True)
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'PUT'})
- self.app.update_request(req)
- res = controller.PUT(req)
- self.assertEqual(res.status_int, 201)
-
- def test_error_limiting(self):
- with save_globals():
- controller = proxy_server.ContainerController(self.app, 'account',
- 'container')
- container_ring = controller.app.container_ring
- controller.app.sort_nodes = lambda l: l
- self.assert_status_map(controller.HEAD, (200, 503, 200, 200), 200,
- missing_container=False)
-
- self.assertEqual(
- node_error_count(controller.app, container_ring.devs[0]), 2)
- self.assertTrue(
- node_last_error(controller.app, container_ring.devs[0])
- is not None)
- for _junk in range(self.app.error_suppression_limit):
- self.assert_status_map(controller.HEAD,
- (200, 503, 503, 503), 503)
- self.assertEqual(
- node_error_count(controller.app, container_ring.devs[0]),
- self.app.error_suppression_limit + 1)
- self.assert_status_map(controller.HEAD, (200, 200, 200, 200), 503)
- self.assertTrue(
- node_last_error(controller.app, container_ring.devs[0])
- is not None)
- self.assert_status_map(controller.PUT, (200, 201, 201, 201), 503,
- missing_container=True)
- self.assert_status_map(controller.DELETE,
- (200, 204, 204, 204), 503)
- self.app.error_suppression_interval = -300
- self.assert_status_map(controller.HEAD, (200, 200, 200, 200), 200)
- self.assert_status_map(controller.DELETE, (200, 204, 204, 204),
- 404, raise_exc=True)
-
- def test_DELETE(self):
- with save_globals():
- controller = proxy_server.ContainerController(self.app, 'account',
- 'container')
- self.assert_status_map(controller.DELETE,
- (200, 204, 204, 204), 204)
- self.assert_status_map(controller.DELETE,
- (200, 204, 204, 503), 204)
- self.assert_status_map(controller.DELETE,
- (200, 204, 503, 503), 503)
- self.assert_status_map(controller.DELETE,
- (200, 204, 404, 404), 404)
- self.assert_status_map(controller.DELETE,
- (200, 404, 404, 404), 404)
- self.assert_status_map(controller.DELETE,
- (200, 204, 503, 404), 503)
-
- self.app.memcache = FakeMemcacheReturnsNone()
- # 200: Account check, 404x3: Container check
- self.assert_status_map(controller.DELETE,
- (200, 404, 404, 404), 404)
-
- def test_response_get_accept_ranges_header(self):
- with save_globals():
- set_http_connect(200, 200, body='{}')
- controller = proxy_server.ContainerController(self.app, 'account',
- 'container')
- req = Request.blank('/v1/a/c?format=json')
- self.app.update_request(req)
- res = controller.GET(req)
- self.assertIn('accept-ranges', res.headers)
- self.assertEqual(res.headers['accept-ranges'], 'bytes')
-
- def test_response_head_accept_ranges_header(self):
- with save_globals():
- set_http_connect(200, 200, body='{}')
- controller = proxy_server.ContainerController(self.app, 'account',
- 'container')
- req = Request.blank('/v1/a/c?format=json')
- self.app.update_request(req)
- res = controller.HEAD(req)
- self.assertIn('accept-ranges', res.headers)
- self.assertEqual(res.headers['accept-ranges'], 'bytes')
-
- def test_PUT_metadata(self):
- self.metadata_helper('PUT')
-
- def test_POST_metadata(self):
- self.metadata_helper('POST')
-
- def metadata_helper(self, method):
- for test_header, test_value in (
- ('X-Container-Meta-TestHeader', 'TestValue'),
- ('X-Container-Meta-TestHeader', ''),
- ('X-Remove-Container-Meta-TestHeader', 'anything'),
- ('X-Container-Read', '.r:*'),
- ('X-Remove-Container-Read', 'anything'),
- ('X-Container-Write', 'anyone'),
- ('X-Remove-Container-Write', 'anything')):
- test_errors = []
-
- def test_connect(ipaddr, port, device, partition, method, path,
- headers=None, query_string=None):
- if path == '/a/c':
- find_header = test_header
- find_value = test_value
- if find_header.lower().startswith('x-remove-'):
- find_header = \
- find_header.lower().replace('-remove', '', 1)
- find_value = ''
- for k, v in headers.items():
- if k.lower() == find_header.lower() and \
- v == find_value:
- break
- else:
- test_errors.append('%s: %s not in %s' %
- (find_header, find_value, headers))
- with save_globals():
- controller = \
- proxy_server.ContainerController(self.app, 'a', 'c')
- set_http_connect(200, 201, 201, 201, give_connect=test_connect)
- req = Request.blank(
- '/v1/a/c',
- environ={'REQUEST_METHOD': method, 'swift_owner': True},
- headers={test_header: test_value})
- self.app.update_request(req)
- getattr(controller, method)(req)
- self.assertEqual(test_errors, [])
-
- def test_PUT_bad_metadata(self):
- self.bad_metadata_helper('PUT')
-
- def test_POST_bad_metadata(self):
- self.bad_metadata_helper('POST')
-
- def bad_metadata_helper(self, method):
- with save_globals():
- controller = proxy_server.ContainerController(self.app, 'a', 'c')
- set_http_connect(200, 201, 201, 201)
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method})
- self.app.update_request(req)
- resp = getattr(controller, method)(req)
- self.assertEqual(resp.status_int, 201)
-
- set_http_connect(201, 201, 201)
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
- headers={'X-Container-Meta-' +
- ('a' * constraints.MAX_META_NAME_LENGTH): 'v'})
- self.app.update_request(req)
- resp = getattr(controller, method)(req)
- self.assertEqual(resp.status_int, 201)
- set_http_connect(201, 201, 201)
- req = Request.blank(
- '/v1/a/c', environ={'REQUEST_METHOD': method},
- headers={'X-Container-Meta-' +
- ('a' * (constraints.MAX_META_NAME_LENGTH + 1)): 'v'})
- self.app.update_request(req)
- resp = getattr(controller, method)(req)
- self.assertEqual(resp.status_int, 400)
-
- set_http_connect(201, 201, 201)
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
- headers={'X-Container-Meta-Too-Long':
- 'a' * constraints.MAX_META_VALUE_LENGTH})
- self.app.update_request(req)
- resp = getattr(controller, method)(req)
- self.assertEqual(resp.status_int, 201)
- set_http_connect(201, 201, 201)
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
- headers={'X-Container-Meta-Too-Long':
- 'a' * (constraints.MAX_META_VALUE_LENGTH + 1)})
- self.app.update_request(req)
- resp = getattr(controller, method)(req)
- self.assertEqual(resp.status_int, 400)
-
- set_http_connect(201, 201, 201)
- headers = {}
- for x in range(constraints.MAX_META_COUNT):
- headers['X-Container-Meta-%d' % x] = 'v'
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
- headers=headers)
- self.app.update_request(req)
- resp = getattr(controller, method)(req)
- self.assertEqual(resp.status_int, 201)
- set_http_connect(201, 201, 201)
- headers = {}
- for x in range(constraints.MAX_META_COUNT + 1):
- headers['X-Container-Meta-%d' % x] = 'v'
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
- headers=headers)
- self.app.update_request(req)
- resp = getattr(controller, method)(req)
- self.assertEqual(resp.status_int, 400)
-
- set_http_connect(201, 201, 201)
- headers = {}
- header_value = 'a' * constraints.MAX_META_VALUE_LENGTH
- size = 0
- x = 0
- while size < (constraints.MAX_META_OVERALL_SIZE - 4
- - constraints.MAX_META_VALUE_LENGTH):
- size += 4 + constraints.MAX_META_VALUE_LENGTH
- headers['X-Container-Meta-%04d' % x] = header_value
- x += 1
- if constraints.MAX_META_OVERALL_SIZE - size > 1:
- headers['X-Container-Meta-a'] = \
- 'a' * (constraints.MAX_META_OVERALL_SIZE - size - 1)
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
- headers=headers)
- self.app.update_request(req)
- resp = getattr(controller, method)(req)
- self.assertEqual(resp.status_int, 201)
- set_http_connect(201, 201, 201)
- headers['X-Container-Meta-a'] = \
- 'a' * (constraints.MAX_META_OVERALL_SIZE - size)
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
- headers=headers)
- self.app.update_request(req)
- resp = getattr(controller, method)(req)
- self.assertEqual(resp.status_int, 400)
-
- def test_POST_calls_clean_acl(self):
- called = [False]
-
- def clean_acl(header, value):
- called[0] = True
- raise ValueError('fake error')
- with save_globals():
- set_http_connect(200, 201, 201, 201)
- controller = proxy_server.ContainerController(self.app, 'account',
- 'container')
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'POST'},
- headers={'X-Container-Read': '.r:*'})
- req.environ['swift.clean_acl'] = clean_acl
- self.app.update_request(req)
- controller.POST(req)
- self.assertTrue(called[0])
- called[0] = False
- with save_globals():
- set_http_connect(200, 201, 201, 201)
- controller = proxy_server.ContainerController(self.app, 'account',
- 'container')
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'POST'},
- headers={'X-Container-Write': '.r:*'})
- req.environ['swift.clean_acl'] = clean_acl
- self.app.update_request(req)
- controller.POST(req)
- self.assertTrue(called[0])
-
- def test_PUT_calls_clean_acl(self):
- called = [False]
-
- def clean_acl(header, value):
- called[0] = True
- raise ValueError('fake error')
- with save_globals():
- set_http_connect(200, 201, 201, 201)
- controller = proxy_server.ContainerController(self.app, 'account',
- 'container')
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'PUT'},
- headers={'X-Container-Read': '.r:*'})
- req.environ['swift.clean_acl'] = clean_acl
- self.app.update_request(req)
- controller.PUT(req)
- self.assertTrue(called[0])
- called[0] = False
- with save_globals():
- set_http_connect(200, 201, 201, 201)
- controller = proxy_server.ContainerController(self.app, 'account',
- 'container')
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'PUT'},
- headers={'X-Container-Write': '.r:*'})
- req.environ['swift.clean_acl'] = clean_acl
- self.app.update_request(req)
- controller.PUT(req)
- self.assertTrue(called[0])
-
- def test_GET_no_content(self):
- with save_globals():
- set_http_connect(200, 204, 204, 204)
- controller = proxy_server.ContainerController(self.app, 'a', 'c')
- req = Request.blank('/v1/a/c')
- self.app.update_request(req)
- res = controller.GET(req)
- self.assertEqual(res.status_int, 204)
- ic = res.environ['swift.infocache']
- self.assertEqual(ic['container/a/c']['status'], 204)
- self.assertEqual(res.content_length, 0)
- self.assertNotIn('transfer-encoding', res.headers)
-
- def test_GET_calls_authorize(self):
- called = [False]
-
- def authorize(req):
- called[0] = True
- return HTTPUnauthorized(request=req)
- with save_globals():
- set_http_connect(200, 201, 201, 201)
- controller = proxy_server.ContainerController(self.app, 'a', 'c')
- req = Request.blank('/v1/a/c')
- req.environ['swift.authorize'] = authorize
- self.app.update_request(req)
- res = controller.GET(req)
- self.assertEqual(
- res.environ['swift.infocache']['container/a/c']['status'],
- 201)
- self.assertTrue(called[0])
-
- def test_HEAD_calls_authorize(self):
- called = [False]
-
- def authorize(req):
- called[0] = True
- return HTTPUnauthorized(request=req)
- with save_globals():
- set_http_connect(200, 201, 201, 201)
- controller = proxy_server.ContainerController(self.app, 'a', 'c')
- req = Request.blank('/v1/a/c', {'REQUEST_METHOD': 'HEAD'})
- req.environ['swift.authorize'] = authorize
- self.app.update_request(req)
- controller.HEAD(req)
- self.assertTrue(called[0])
-
- def test_unauthorized_requests_when_account_not_found(self):
- # verify unauthorized container requests always return response
- # from swift.authorize
- called = [0, 0]
-
- def authorize(req):
- called[0] += 1
- return HTTPUnauthorized(request=req)
-
- def account_info(*args):
- called[1] += 1
- return None, None, None
-
- def _do_test(method):
- with save_globals():
- swift.proxy.controllers.Controller.account_info = account_info
- app = proxy_server.Application(None, FakeMemcache(),
- account_ring=FakeRing(),
- container_ring=FakeRing())
- set_http_connect(201, 201, 201)
- req = Request.blank('/v1/a/c', {'REQUEST_METHOD': method})
- req.environ['swift.authorize'] = authorize
- self.app.update_request(req)
- res = app.handle_request(req)
- return res
-
- for method in ('PUT', 'POST', 'DELETE'):
- # no delay_denial on method, expect one call to authorize
- called = [0, 0]
- res = _do_test(method)
- self.assertEqual(401, res.status_int)
- self.assertEqual([1, 0], called)
-
- for method in ('HEAD', 'GET'):
- # delay_denial on method, expect two calls to authorize
- called = [0, 0]
- res = _do_test(method)
- self.assertEqual(401, res.status_int)
- self.assertEqual([2, 1], called)
-
- def test_authorized_requests_when_account_not_found(self):
- # verify authorized container requests always return 404 when
- # account not found
- called = [0, 0]
-
- def authorize(req):
- called[0] += 1
-
- def account_info(*args):
- called[1] += 1
- return None, None, None
-
- def _do_test(method):
- with save_globals():
- swift.proxy.controllers.Controller.account_info = account_info
- app = proxy_server.Application(None, FakeMemcache(),
- account_ring=FakeRing(),
- container_ring=FakeRing())
- set_http_connect(201, 201, 201)
- req = Request.blank('/v1/a/c', {'REQUEST_METHOD': method})
- req.environ['swift.authorize'] = authorize
- self.app.update_request(req)
- res = app.handle_request(req)
- return res
-
- for method in ('PUT', 'POST', 'DELETE', 'HEAD', 'GET'):
- # expect one call to authorize
- called = [0, 0]
- res = _do_test(method)
- self.assertEqual(404, res.status_int)
- self.assertEqual([1, 1], called)
-
- def test_OPTIONS_get_info_drops_origin(self):
- with save_globals():
- controller = proxy_server.ContainerController(self.app, 'a', 'c')
-
- count = [0]
-
- def my_get_info(app, env, account, container=None,
- ret_not_found=False, swift_source=None):
- if count[0] > 11:
- return {}
- count[0] += 1
- if not container:
- return {'some': 'stuff'}
- return proxy_base.was_get_info(
- app, env, account, container, ret_not_found, swift_source)
-
- proxy_base.was_get_info = proxy_base.get_info
- with mock.patch.object(proxy_base, 'get_info', my_get_info):
- proxy_base.get_info = my_get_info
- req = Request.blank(
- '/v1/a/c',
- {'REQUEST_METHOD': 'OPTIONS'},
- headers={'Origin': 'http://foo.com',
- 'Access-Control-Request-Method': 'GET'})
- controller.OPTIONS(req)
- self.assertLess(count[0], 11)
-
- def test_OPTIONS(self):
- with save_globals():
- controller = proxy_server.ContainerController(self.app, 'a', 'c')
-
- def my_empty_container_info(*args):
- return {}
- controller.container_info = my_empty_container_info
- req = Request.blank(
- '/v1/a/c',
- {'REQUEST_METHOD': 'OPTIONS'},
- headers={'Origin': 'http://foo.com',
- 'Access-Control-Request-Method': 'GET'})
- resp = controller.OPTIONS(req)
- self.assertEqual(401, resp.status_int)
-
- def my_empty_origin_container_info(*args):
- return {'cors': {'allow_origin': None}}
- controller.container_info = my_empty_origin_container_info
- req = Request.blank(
- '/v1/a/c',
- {'REQUEST_METHOD': 'OPTIONS'},
- headers={'Origin': 'http://foo.com',
- 'Access-Control-Request-Method': 'GET'})
- resp = controller.OPTIONS(req)
- self.assertEqual(401, resp.status_int)
-
- def my_container_info(*args):
- return {
- 'cors': {
- 'allow_origin': 'http://foo.bar:8080 https://foo.bar',
- 'max_age': '999',
- }
- }
- controller.container_info = my_container_info
- req = Request.blank(
- '/v1/a/c',
- {'REQUEST_METHOD': 'OPTIONS'},
- headers={'Origin': 'https://foo.bar',
- 'Access-Control-Request-Method': 'GET'})
- req.content_length = 0
- resp = controller.OPTIONS(req)
- self.assertEqual(200, resp.status_int)
- self.assertEqual(
- 'https://foo.bar',
- resp.headers['access-control-allow-origin'])
- for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split():
- self.assertIn(verb,
- resp.headers['access-control-allow-methods'])
- self.assertEqual(
- len(resp.headers['access-control-allow-methods'].split(', ')),
- 6)
- self.assertEqual('999', resp.headers['access-control-max-age'])
- req = Request.blank(
- '/v1/a/c',
- {'REQUEST_METHOD': 'OPTIONS'},
- headers={'Origin': 'https://foo.bar'})
- req.content_length = 0
- resp = controller.OPTIONS(req)
- self.assertEqual(401, resp.status_int)
- req = Request.blank('/v1/a/c', {'REQUEST_METHOD': 'OPTIONS'})
- req.content_length = 0
- resp = controller.OPTIONS(req)
- self.assertEqual(200, resp.status_int)
- for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split():
- self.assertIn(verb, resp.headers['Allow'])
- self.assertEqual(len(resp.headers['Allow'].split(', ')), 6)
- req = Request.blank(
- '/v1/a/c',
- {'REQUEST_METHOD': 'OPTIONS'},
- headers={'Origin': 'http://foo.bar',
- 'Access-Control-Request-Method': 'GET'})
- resp = controller.OPTIONS(req)
- self.assertEqual(401, resp.status_int)
- req = Request.blank(
- '/v1/a/c',
- {'REQUEST_METHOD': 'OPTIONS'},
- headers={'Origin': 'http://foo.bar',
- 'Access-Control-Request-Method': 'GET'})
- controller.app.cors_allow_origin = ['http://foo.bar', ]
- resp = controller.OPTIONS(req)
- self.assertEqual(200, resp.status_int)
-
- def my_container_info_wildcard(*args):
- return {
- 'cors': {
- 'allow_origin': '*',
- 'max_age': '999',
- }
- }
- controller.container_info = my_container_info_wildcard
- req = Request.blank(
- '/v1/a/c/o.jpg',
- {'REQUEST_METHOD': 'OPTIONS'},
- headers={'Origin': 'https://bar.baz',
- 'Access-Control-Request-Method': 'GET'})
- req.content_length = 0
- resp = controller.OPTIONS(req)
- self.assertEqual(200, resp.status_int)
- self.assertEqual('*', resp.headers['access-control-allow-origin'])
- for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split():
- self.assertIn(verb,
- resp.headers['access-control-allow-methods'])
- self.assertEqual(
- len(resp.headers['access-control-allow-methods'].split(', ')),
- 6)
- self.assertEqual('999', resp.headers['access-control-max-age'])
-
- req = Request.blank(
- '/v1/a/c/o.jpg',
- {'REQUEST_METHOD': 'OPTIONS'},
- headers={'Origin': 'https://bar.baz',
- 'Access-Control-Request-Headers':
- 'x-foo, x-bar, x-auth-token',
- 'Access-Control-Request-Method': 'GET'}
- )
- req.content_length = 0
- resp = controller.OPTIONS(req)
- self.assertEqual(200, resp.status_int)
- self.assertEqual(
- sortHeaderNames('x-foo, x-bar, x-auth-token'),
- sortHeaderNames(resp.headers['access-control-allow-headers']))
-
- def test_CORS_valid(self):
- with save_globals():
- controller = proxy_server.ContainerController(self.app, 'a', 'c')
-
- def stubContainerInfo(*args):
- return {
- 'cors': {
- 'allow_origin': 'http://foo.bar'
- }
- }
- controller.container_info = stubContainerInfo
-
- def containerGET(controller, req):
- return Response(headers={
- 'X-Container-Meta-Color': 'red',
- 'X-Super-Secret': 'hush',
- })
-
- req = Request.blank(
- '/v1/a/c',
- {'REQUEST_METHOD': 'GET'},
- headers={'Origin': 'http://foo.bar'})
-
- resp = cors_validation(containerGET)(controller, req)
-
- self.assertEqual(200, resp.status_int)
- self.assertEqual('http://foo.bar',
- resp.headers['access-control-allow-origin'])
- self.assertEqual('red', resp.headers['x-container-meta-color'])
- # X-Super-Secret is in the response, but not "exposed"
- self.assertEqual('hush', resp.headers['x-super-secret'])
- self.assertIn('access-control-expose-headers', resp.headers)
- exposed = set(
- h.strip() for h in
- resp.headers['access-control-expose-headers'].split(','))
- expected_exposed = set([
- 'cache-control', 'content-language', 'content-type', 'expires',
- 'last-modified', 'pragma', 'etag', 'x-timestamp', 'x-trans-id',
- 'x-openstack-request-id', 'x-container-meta-color'])
- self.assertEqual(expected_exposed, exposed)
-
- def _gather_x_account_headers(self, controller_call, req, *connect_args,
- **kwargs):
- seen_headers = []
- to_capture = ('X-Account-Partition', 'X-Account-Host',
- 'X-Account-Device')
-
- def capture_headers(ipaddr, port, device, partition, method,
- path, headers=None, query_string=None):
- captured = {}
- for header in to_capture:
- captured[header] = headers.get(header)
- seen_headers.append(captured)
-
- with save_globals():
- self.app.allow_account_management = True
-
- set_http_connect(*connect_args, give_connect=capture_headers,
- **kwargs)
- resp = controller_call(req)
- self.assertEqual(2, resp.status_int // 100) # sanity check
-
- # don't care about the account HEAD, so throw away the
- # first element
- return sorted(seen_headers[1:],
- key=lambda d: d['X-Account-Host'] or 'Z')
-
- def test_PUT_x_account_headers_with_fewer_account_replicas(self):
- self.app.account_ring.set_replicas(2)
- req = Request.blank('/v1/a/c', headers={'': ''})
- controller = proxy_server.ContainerController(self.app, 'a', 'c')
-
- seen_headers = self._gather_x_account_headers(
- controller.PUT, req,
- 200, 201, 201, 201) # HEAD PUT PUT PUT
- self.assertEqual(seen_headers, [
- {'X-Account-Host': '10.0.0.0:1000',
- 'X-Account-Partition': '0',
- 'X-Account-Device': 'sda'},
- {'X-Account-Host': '10.0.0.1:1001',
- 'X-Account-Partition': '0',
- 'X-Account-Device': 'sdb'},
- {'X-Account-Host': None,
- 'X-Account-Partition': None,
- 'X-Account-Device': None}
- ])
-
- def test_PUT_x_account_headers_with_more_account_replicas(self):
- self.app.account_ring.set_replicas(4)
- req = Request.blank('/v1/a/c', headers={'': ''})
- controller = proxy_server.ContainerController(self.app, 'a', 'c')
-
- seen_headers = self._gather_x_account_headers(
- controller.PUT, req,
- 200, 201, 201, 201) # HEAD PUT PUT PUT
- self.assertEqual(seen_headers, [
- {'X-Account-Host': '10.0.0.0:1000,10.0.0.3:1003',
- 'X-Account-Partition': '0',
- 'X-Account-Device': 'sda,sdd'},
- {'X-Account-Host': '10.0.0.1:1001',
- 'X-Account-Partition': '0',
- 'X-Account-Device': 'sdb'},
- {'X-Account-Host': '10.0.0.2:1002',
- 'X-Account-Partition': '0',
- 'X-Account-Device': 'sdc'}
- ])
-
- def test_DELETE_x_account_headers_with_fewer_account_replicas(self):
- self.app.account_ring.set_replicas(2)
- req = Request.blank('/v1/a/c', headers={'': ''})
- controller = proxy_server.ContainerController(self.app, 'a', 'c')
-
- seen_headers = self._gather_x_account_headers(
- controller.DELETE, req,
- 200, 204, 204, 204) # HEAD DELETE DELETE DELETE
- self.assertEqual(seen_headers, [
- {'X-Account-Host': '10.0.0.0:1000',
- 'X-Account-Partition': '0',
- 'X-Account-Device': 'sda'},
- {'X-Account-Host': '10.0.0.1:1001',
- 'X-Account-Partition': '0',
- 'X-Account-Device': 'sdb'},
- {'X-Account-Host': None,
- 'X-Account-Partition': None,
- 'X-Account-Device': None}
- ])
-
- def test_DELETE_x_account_headers_with_more_account_replicas(self):
- self.app.account_ring.set_replicas(4)
- req = Request.blank('/v1/a/c', headers={'': ''})
- controller = proxy_server.ContainerController(self.app, 'a', 'c')
-
- seen_headers = self._gather_x_account_headers(
- controller.DELETE, req,
- 200, 204, 204, 204) # HEAD DELETE DELETE DELETE
- self.assertEqual(seen_headers, [
- {'X-Account-Host': '10.0.0.0:1000,10.0.0.3:1003',
- 'X-Account-Partition': '0',
- 'X-Account-Device': 'sda,sdd'},
- {'X-Account-Host': '10.0.0.1:1001',
- 'X-Account-Partition': '0',
- 'X-Account-Device': 'sdb'},
- {'X-Account-Host': '10.0.0.2:1002',
- 'X-Account-Partition': '0',
- 'X-Account-Device': 'sdc'}
- ])
-
- def test_PUT_backed_x_timestamp_header(self):
- timestamps = []
-
- def capture_timestamps(*args, **kwargs):
- headers = kwargs['headers']
- timestamps.append(headers.get('X-Timestamp'))
-
- req = Request.blank('/v1/a/c', method='PUT', headers={'': ''})
- with save_globals():
- new_connect = set_http_connect(200, # account existence check
- 201, 201, 201,
- give_connect=capture_timestamps)
- resp = self.app.handle_request(req)
-
- # sanity
- self.assertRaises(StopIteration, new_connect.code_iter.next)
- self.assertEqual(2, resp.status_int // 100)
-
- timestamps.pop(0) # account existence check
- self.assertEqual(3, len(timestamps))
- for timestamp in timestamps:
- self.assertEqual(timestamp, timestamps[0])
- self.assertTrue(re.match('[0-9]{10}\.[0-9]{5}', timestamp))
-
- def test_DELETE_backed_x_timestamp_header(self):
- timestamps = []
-
- def capture_timestamps(*args, **kwargs):
- headers = kwargs['headers']
- timestamps.append(headers.get('X-Timestamp'))
-
- req = Request.blank('/v1/a/c', method='DELETE', headers={'': ''})
- self.app.update_request(req)
- with save_globals():
- new_connect = set_http_connect(200, # account existence check
- 201, 201, 201,
- give_connect=capture_timestamps)
- resp = self.app.handle_request(req)
-
- # sanity
- self.assertRaises(StopIteration, new_connect.code_iter.next)
- self.assertEqual(2, resp.status_int // 100)
-
- timestamps.pop(0) # account existence check
- self.assertEqual(3, len(timestamps))
- for timestamp in timestamps:
- self.assertEqual(timestamp, timestamps[0])
- self.assertTrue(re.match('[0-9]{10}\.[0-9]{5}', timestamp))
-
- def test_node_read_timeout_retry_to_container(self):
- with save_globals():
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': 'GET'})
- self.app.node_timeout = 0.1
- set_http_connect(200, 200, 200, body='abcdef', slow=[1.0, 1.0])
- resp = req.get_response(self.app)
- got_exc = False
- try:
- resp.body
- except ChunkReadTimeout:
- got_exc = True
- self.assertTrue(got_exc)
-
-
-@patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())])
-class TestAccountController(unittest.TestCase):
-
- def setUp(self):
- self.app = proxy_server.Application(None, FakeMemcache(),
- account_ring=FakeRing(),
- container_ring=FakeRing())
-
- def assert_status_map(self, method, statuses, expected, env_expected=None,
- headers=None, **kwargs):
- headers = headers or {}
- with save_globals():
- set_http_connect(*statuses, **kwargs)
- req = Request.blank('/v1/a', {}, headers=headers)
- self.app.update_request(req)
- res = method(req)
- self.assertEqual(res.status_int, expected)
- infocache = res.environ.get('swift.infocache', {})
- if env_expected:
- self.assertEqual(infocache['account/a']['status'],
- env_expected)
- set_http_connect(*statuses)
- req = Request.blank('/v1/a/', {})
- self.app.update_request(req)
- res = method(req)
- infocache = res.environ.get('swift.infocache', {})
- self.assertEqual(res.status_int, expected)
- if env_expected:
- self.assertEqual(infocache['account/a']['status'],
- env_expected)
-
- def test_OPTIONS(self):
- with save_globals():
- self.app.allow_account_management = False
- controller = proxy_server.AccountController(self.app, 'account')
- req = Request.blank('/v1/account', {'REQUEST_METHOD': 'OPTIONS'})
- req.content_length = 0
- resp = controller.OPTIONS(req)
- self.assertEqual(200, resp.status_int)
- for verb in 'OPTIONS GET POST HEAD'.split():
- self.assertIn(verb, resp.headers['Allow'])
- self.assertEqual(len(resp.headers['Allow'].split(', ')), 4)
-
- # Test a CORS OPTIONS request (i.e. including Origin and
- # Access-Control-Request-Method headers)
- self.app.allow_account_management = False
- controller = proxy_server.AccountController(self.app, 'account')
- req = Request.blank(
- '/v1/account', {'REQUEST_METHOD': 'OPTIONS'},
- headers={'Origin': 'http://foo.com',
- 'Access-Control-Request-Method': 'GET'})
- req.content_length = 0
- resp = controller.OPTIONS(req)
- self.assertEqual(200, resp.status_int)
- for verb in 'OPTIONS GET POST HEAD'.split():
- self.assertIn(verb, resp.headers['Allow'])
- self.assertEqual(len(resp.headers['Allow'].split(', ')), 4)
-
- self.app.allow_account_management = True
- controller = proxy_server.AccountController(self.app, 'account')
- req = Request.blank('/v1/account', {'REQUEST_METHOD': 'OPTIONS'})
- req.content_length = 0
- resp = controller.OPTIONS(req)
- self.assertEqual(200, resp.status_int)
- for verb in 'OPTIONS GET POST PUT DELETE HEAD'.split():
- self.assertIn(verb, resp.headers['Allow'])
- self.assertEqual(len(resp.headers['Allow'].split(', ')), 6)
-
- def test_GET(self):
- with save_globals():
- controller = proxy_server.AccountController(self.app, 'a')
- # GET returns after the first successful call to an Account Server
- self.assert_status_map(controller.GET, (200,), 200, 200)
- self.assert_status_map(controller.GET, (503, 200), 200, 200)
- self.assert_status_map(controller.GET, (503, 503, 200), 200, 200)
- self.assert_status_map(controller.GET, (204,), 204, 204)
- self.assert_status_map(controller.GET, (503, 204), 204, 204)
- self.assert_status_map(controller.GET, (503, 503, 204), 204, 204)
- self.assert_status_map(controller.GET, (404, 200), 200, 200)
- self.assert_status_map(controller.GET, (404, 404, 200), 200, 200)
- self.assert_status_map(controller.GET, (404, 503, 204), 204, 204)
- # If Account servers fail, if autocreate = False, return majority
- # response
- self.assert_status_map(controller.GET, (404, 404, 404), 404, 404)
- self.assert_status_map(controller.GET, (404, 404, 503), 404, 404)
- self.assert_status_map(controller.GET, (404, 503, 503), 503)
-
- self.app.memcache = FakeMemcacheReturnsNone()
- self.assert_status_map(controller.GET, (404, 404, 404), 404, 404)
-
- def test_GET_autocreate(self):
- with save_globals():
- controller = proxy_server.AccountController(self.app, 'a')
- self.app.memcache = FakeMemcacheReturnsNone()
- self.assertFalse(self.app.account_autocreate)
- # Repeat the test for autocreate = False and 404 by all
- self.assert_status_map(controller.GET,
- (404, 404, 404), 404)
- self.assert_status_map(controller.GET,
- (404, 503, 404), 404)
- # When autocreate is True, if none of the nodes respond 2xx
- # And quorum of the nodes responded 404,
- # ALL nodes are asked to create the account
- # If successful, the GET request is repeated.
- controller.app.account_autocreate = True
- self.assert_status_map(controller.GET,
- (404, 404, 404), 204)
- self.assert_status_map(controller.GET,
- (404, 503, 404), 204)
-
- # We always return 503 if no majority between 4xx, 3xx or 2xx found
- self.assert_status_map(controller.GET,
- (500, 500, 400), 503)
-
- def test_HEAD(self):
- # Same behaviour as GET
- with save_globals():
- controller = proxy_server.AccountController(self.app, 'a')
- self.assert_status_map(controller.HEAD, (200,), 200, 200)
- self.assert_status_map(controller.HEAD, (503, 200), 200, 200)
- self.assert_status_map(controller.HEAD, (503, 503, 200), 200, 200)
- self.assert_status_map(controller.HEAD, (204,), 204, 204)
- self.assert_status_map(controller.HEAD, (503, 204), 204, 204)
- self.assert_status_map(controller.HEAD, (204, 503, 503), 204, 204)
- self.assert_status_map(controller.HEAD, (204,), 204, 204)
- self.assert_status_map(controller.HEAD, (404, 404, 404), 404, 404)
- self.assert_status_map(controller.HEAD, (404, 404, 200), 200, 200)
- self.assert_status_map(controller.HEAD, (404, 200), 200, 200)
- self.assert_status_map(controller.HEAD, (404, 404, 503), 404, 404)
- self.assert_status_map(controller.HEAD, (404, 503, 503), 503)
- self.assert_status_map(controller.HEAD, (404, 503, 204), 204, 204)
-
- def test_HEAD_autocreate(self):
- # Same behaviour as GET
- with save_globals():
- controller = proxy_server.AccountController(self.app, 'a')
- self.app.memcache = FakeMemcacheReturnsNone()
- self.assertFalse(self.app.account_autocreate)
- self.assert_status_map(controller.HEAD,
- (404, 404, 404), 404)
- controller.app.account_autocreate = True
- self.assert_status_map(controller.HEAD,
- (404, 404, 404), 204)
- self.assert_status_map(controller.HEAD,
- (500, 404, 404), 204)
- # We always return 503 if no majority between 4xx, 3xx or 2xx found
- self.assert_status_map(controller.HEAD,
- (500, 500, 400), 503)
-
- def test_POST_autocreate(self):
- with save_globals():
- controller = proxy_server.AccountController(self.app, 'a')
- self.app.memcache = FakeMemcacheReturnsNone()
- # first test with autocreate being False
- self.assertFalse(self.app.account_autocreate)
- self.assert_status_map(controller.POST,
- (404, 404, 404), 404)
- # next turn it on and test account being created than updated
- controller.app.account_autocreate = True
- self.assert_status_map(
- controller.POST,
- (404, 404, 404, 202, 202, 202, 201, 201, 201), 201)
- # account_info PUT account POST account
- self.assert_status_map(
- controller.POST,
- (404, 404, 503, 201, 201, 503, 204, 204, 504), 204)
- # what if create fails
- self.assert_status_map(
- controller.POST,
- (404, 404, 404, 403, 403, 403, 400, 400, 400), 400)
-
- def test_POST_autocreate_with_sysmeta(self):
- with save_globals():
- controller = proxy_server.AccountController(self.app, 'a')
- self.app.memcache = FakeMemcacheReturnsNone()
- # first test with autocreate being False
- self.assertFalse(self.app.account_autocreate)
- self.assert_status_map(controller.POST,
- (404, 404, 404), 404)
- # next turn it on and test account being created than updated
- controller.app.account_autocreate = True
- calls = []
- callback = _make_callback_func(calls)
- key, value = 'X-Account-Sysmeta-Blah', 'something'
- headers = {key: value}
- self.assert_status_map(
- controller.POST,
- (404, 404, 404, 202, 202, 202, 201, 201, 201), 201,
- # POST , autocreate PUT, POST again
- headers=headers,
- give_connect=callback)
- self.assertEqual(9, len(calls))
- for call in calls:
- self.assertIn(key, call['headers'],
- '%s call, key %s missing in headers %s' %
- (call['method'], key, call['headers']))
- self.assertEqual(value, call['headers'][key])
-
- def test_connection_refused(self):
- self.app.account_ring.get_nodes('account')
- for dev in self.app.account_ring.devs:
- dev['ip'] = '127.0.0.1'
- dev['port'] = 1 # can't connect on this port
- controller = proxy_server.AccountController(self.app, 'account')
- req = Request.blank('/v1/account', environ={'REQUEST_METHOD': 'HEAD'})
- self.app.update_request(req)
- resp = controller.HEAD(req)
- self.assertEqual(resp.status_int, 503)
-
- def test_other_socket_error(self):
- self.app.account_ring.get_nodes('account')
- for dev in self.app.account_ring.devs:
- dev['ip'] = '127.0.0.1'
- dev['port'] = -1 # invalid port number
- controller = proxy_server.AccountController(self.app, 'account')
- req = Request.blank('/v1/account', environ={'REQUEST_METHOD': 'HEAD'})
- self.app.update_request(req)
- resp = controller.HEAD(req)
- self.assertEqual(resp.status_int, 503)
-
- def test_response_get_accept_ranges_header(self):
- with save_globals():
- set_http_connect(200, 200, body='{}')
- controller = proxy_server.AccountController(self.app, 'account')
- req = Request.blank('/v1/a?format=json')
- self.app.update_request(req)
- res = controller.GET(req)
- self.assertIn('accept-ranges', res.headers)
- self.assertEqual(res.headers['accept-ranges'], 'bytes')
-
- def test_response_head_accept_ranges_header(self):
- with save_globals():
- set_http_connect(200, 200, body='{}')
- controller = proxy_server.AccountController(self.app, 'account')
- req = Request.blank('/v1/a?format=json')
- self.app.update_request(req)
- res = controller.HEAD(req)
- res.body
- self.assertIn('accept-ranges', res.headers)
- self.assertEqual(res.headers['accept-ranges'], 'bytes')
-
- def test_PUT(self):
- with save_globals():
- controller = proxy_server.AccountController(self.app, 'account')
-
- def test_status_map(statuses, expected, **kwargs):
- set_http_connect(*statuses, **kwargs)
- self.app.memcache.store = {}
- req = Request.blank('/v1/a', {})
- req.content_length = 0
- self.app.update_request(req)
- res = controller.PUT(req)
- expected = str(expected)
- self.assertEqual(res.status[:len(expected)], expected)
- test_status_map((201, 201, 201), 405)
- self.app.allow_account_management = True
- test_status_map((201, 201, 201), 201)
- test_status_map((201, 201, 500), 201)
- test_status_map((201, 500, 500), 503)
- test_status_map((204, 500, 404), 503)
-
- def test_PUT_max_account_name_length(self):
- with save_globals():
- self.app.allow_account_management = True
- limit = constraints.MAX_ACCOUNT_NAME_LENGTH
- controller = proxy_server.AccountController(self.app, '1' * limit)
- self.assert_status_map(controller.PUT, (201, 201, 201), 201)
- controller = proxy_server.AccountController(
- self.app, '2' * (limit + 1))
- self.assert_status_map(controller.PUT, (201, 201, 201), 400)
-
- def test_PUT_connect_exceptions(self):
- with save_globals():
- self.app.allow_account_management = True
- controller = proxy_server.AccountController(self.app, 'account')
- self.assert_status_map(controller.PUT, (201, 201, -1), 201)
- self.assert_status_map(controller.PUT, (201, -1, -1), 503)
- self.assert_status_map(controller.PUT, (503, 503, -1), 503)
-
- def test_PUT_status(self):
- with save_globals():
- self.app.allow_account_management = True
- controller = proxy_server.AccountController(self.app, 'account')
- self.assert_status_map(controller.PUT, (201, 201, 202), 202)
-
- def test_PUT_metadata(self):
- self.metadata_helper('PUT')
-
- def test_POST_metadata(self):
- self.metadata_helper('POST')
-
- def metadata_helper(self, method):
- for test_header, test_value in (
- ('X-Account-Meta-TestHeader', 'TestValue'),
- ('X-Account-Meta-TestHeader', ''),
- ('X-Remove-Account-Meta-TestHeader', 'anything')):
- test_errors = []
-
- def test_connect(ipaddr, port, device, partition, method, path,
- headers=None, query_string=None):
- if path == '/a':
- find_header = test_header
- find_value = test_value
- if find_header.lower().startswith('x-remove-'):
- find_header = \
- find_header.lower().replace('-remove', '', 1)
- find_value = ''
- for k, v in headers.items():
- if k.lower() == find_header.lower() and \
- v == find_value:
- break
- else:
- test_errors.append('%s: %s not in %s' %
- (find_header, find_value, headers))
- with save_globals():
- self.app.allow_account_management = True
- controller = \
- proxy_server.AccountController(self.app, 'a')
- set_http_connect(201, 201, 201, give_connect=test_connect)
- req = Request.blank('/v1/a/c',
- environ={'REQUEST_METHOD': method},
- headers={test_header: test_value})
- self.app.update_request(req)
- getattr(controller, method)(req)
- self.assertEqual(test_errors, [])
-
- def test_PUT_bad_metadata(self):
- self.bad_metadata_helper('PUT')
-
- def test_POST_bad_metadata(self):
- self.bad_metadata_helper('POST')
-
- def bad_metadata_helper(self, method):
- with save_globals():
- self.app.allow_account_management = True
- controller = proxy_server.AccountController(self.app, 'a')
- set_http_connect(200, 201, 201, 201)
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method})
- self.app.update_request(req)
- resp = getattr(controller, method)(req)
- self.assertEqual(resp.status_int, 201)
-
- set_http_connect(201, 201, 201)
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
- headers={'X-Account-Meta-' +
- ('a' * constraints.MAX_META_NAME_LENGTH): 'v'})
- self.app.update_request(req)
- resp = getattr(controller, method)(req)
- self.assertEqual(resp.status_int, 201)
- set_http_connect(201, 201, 201)
- req = Request.blank(
- '/v1/a/c', environ={'REQUEST_METHOD': method},
- headers={'X-Account-Meta-' +
- ('a' * (constraints.MAX_META_NAME_LENGTH + 1)): 'v'})
- self.app.update_request(req)
- resp = getattr(controller, method)(req)
- self.assertEqual(resp.status_int, 400)
-
- set_http_connect(201, 201, 201)
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
- headers={'X-Account-Meta-Too-Long':
- 'a' * constraints.MAX_META_VALUE_LENGTH})
- self.app.update_request(req)
- resp = getattr(controller, method)(req)
- self.assertEqual(resp.status_int, 201)
- set_http_connect(201, 201, 201)
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
- headers={'X-Account-Meta-Too-Long':
- 'a' * (constraints.MAX_META_VALUE_LENGTH + 1)})
- self.app.update_request(req)
- resp = getattr(controller, method)(req)
- self.assertEqual(resp.status_int, 400)
-
- set_http_connect(201, 201, 201)
- headers = {}
- for x in range(constraints.MAX_META_COUNT):
- headers['X-Account-Meta-%d' % x] = 'v'
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
- headers=headers)
- self.app.update_request(req)
- resp = getattr(controller, method)(req)
- self.assertEqual(resp.status_int, 201)
- set_http_connect(201, 201, 201)
- headers = {}
- for x in range(constraints.MAX_META_COUNT + 1):
- headers['X-Account-Meta-%d' % x] = 'v'
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
- headers=headers)
- self.app.update_request(req)
- resp = getattr(controller, method)(req)
- self.assertEqual(resp.status_int, 400)
-
- set_http_connect(201, 201, 201)
- headers = {}
- header_value = 'a' * constraints.MAX_META_VALUE_LENGTH
- size = 0
- x = 0
- while size < (constraints.MAX_META_OVERALL_SIZE - 4
- - constraints.MAX_META_VALUE_LENGTH):
- size += 4 + constraints.MAX_META_VALUE_LENGTH
- headers['X-Account-Meta-%04d' % x] = header_value
- x += 1
- if constraints.MAX_META_OVERALL_SIZE - size > 1:
- headers['X-Account-Meta-a'] = \
- 'a' * (constraints.MAX_META_OVERALL_SIZE - size - 1)
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
- headers=headers)
- self.app.update_request(req)
- resp = getattr(controller, method)(req)
- self.assertEqual(resp.status_int, 201)
- set_http_connect(201, 201, 201)
- headers['X-Account-Meta-a'] = \
- 'a' * (constraints.MAX_META_OVERALL_SIZE - size)
- req = Request.blank('/v1/a/c', environ={'REQUEST_METHOD': method},
- headers=headers)
- self.app.update_request(req)
- resp = getattr(controller, method)(req)
- self.assertEqual(resp.status_int, 400)
-
- def test_DELETE(self):
- with save_globals():
- controller = proxy_server.AccountController(self.app, 'account')
-
- def test_status_map(statuses, expected, **kwargs):
- set_http_connect(*statuses, **kwargs)
- self.app.memcache.store = {}
- req = Request.blank('/v1/a', {'REQUEST_METHOD': 'DELETE'})
- req.content_length = 0
- self.app.update_request(req)
- res = controller.DELETE(req)
- expected = str(expected)
- self.assertEqual(res.status[:len(expected)], expected)
- test_status_map((201, 201, 201), 405)
- self.app.allow_account_management = True
- test_status_map((201, 201, 201), 201)
- test_status_map((201, 201, 500), 201)
- test_status_map((201, 500, 500), 503)
- test_status_map((204, 500, 404), 503)
-
- def test_DELETE_with_query_string(self):
- # Extra safety in case someone typos a query string for an
- # account-level DELETE request that was really meant to be caught by
- # some middleware.
- with save_globals():
- controller = proxy_server.AccountController(self.app, 'account')
-
- def test_status_map(statuses, expected, **kwargs):
- set_http_connect(*statuses, **kwargs)
- self.app.memcache.store = {}
- req = Request.blank('/v1/a?whoops',
- environ={'REQUEST_METHOD': 'DELETE'})
- req.content_length = 0
- self.app.update_request(req)
- res = controller.DELETE(req)
- expected = str(expected)
- self.assertEqual(res.status[:len(expected)], expected)
- test_status_map((201, 201, 201), 400)
- self.app.allow_account_management = True
- test_status_map((201, 201, 201), 400)
- test_status_map((201, 201, 500), 400)
- test_status_map((201, 500, 500), 400)
- test_status_map((204, 500, 404), 400)
-
-
-@patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())])
-class TestAccountControllerFakeGetResponse(unittest.TestCase):
- """
- Test all the faked-out GET responses for accounts that don't exist. They
- have to match the responses for empty accounts that really exist.
- """
- def setUp(self):
- conf = {'account_autocreate': 'yes'}
- self.app = proxy_server.Application(conf, FakeMemcache(),
- account_ring=FakeRing(),
- container_ring=FakeRing())
- self.app.memcache = FakeMemcacheReturnsNone()
-
- def test_GET_autocreate_accept_json(self):
- with save_globals():
- set_http_connect(*([404] * 100)) # nonexistent: all backends 404
- req = Request.blank(
- '/v1/a', headers={'Accept': 'application/json'},
- environ={'REQUEST_METHOD': 'GET',
- 'PATH_INFO': '/v1/a'})
- resp = req.get_response(self.app)
- self.assertEqual(200, resp.status_int)
- self.assertEqual('application/json; charset=utf-8',
- resp.headers['Content-Type'])
- self.assertEqual("[]", resp.body)
-
- def test_GET_autocreate_format_json(self):
- with save_globals():
- set_http_connect(*([404] * 100)) # nonexistent: all backends 404
- req = Request.blank('/v1/a?format=json',
- environ={'REQUEST_METHOD': 'GET',
- 'PATH_INFO': '/v1/a',
- 'QUERY_STRING': 'format=json'})
- resp = req.get_response(self.app)
- self.assertEqual(200, resp.status_int)
- self.assertEqual('application/json; charset=utf-8',
- resp.headers['Content-Type'])
- self.assertEqual("[]", resp.body)
-
- def test_GET_autocreate_accept_xml(self):
- with save_globals():
- set_http_connect(*([404] * 100)) # nonexistent: all backends 404
- req = Request.blank('/v1/a', headers={"Accept": "text/xml"},
- environ={'REQUEST_METHOD': 'GET',
- 'PATH_INFO': '/v1/a'})
-
- resp = req.get_response(self.app)
- self.assertEqual(200, resp.status_int)
-
- self.assertEqual('text/xml; charset=utf-8',
- resp.headers['Content-Type'])
- empty_xml_listing = ('<?xml version="1.0" encoding="UTF-8"?>\n'
- '<account name="a">\n</account>')
- self.assertEqual(empty_xml_listing, resp.body)
-
- def test_GET_autocreate_format_xml(self):
- with save_globals():
- set_http_connect(*([404] * 100)) # nonexistent: all backends 404
- req = Request.blank('/v1/a?format=xml',
- environ={'REQUEST_METHOD': 'GET',
- 'PATH_INFO': '/v1/a',
- 'QUERY_STRING': 'format=xml'})
- resp = req.get_response(self.app)
- self.assertEqual(200, resp.status_int)
- self.assertEqual('application/xml; charset=utf-8',
- resp.headers['Content-Type'])
- empty_xml_listing = ('<?xml version="1.0" encoding="UTF-8"?>\n'
- '<account name="a">\n</account>')
- self.assertEqual(empty_xml_listing, resp.body)
-
- def test_GET_autocreate_accept_unknown(self):
- with save_globals():
- set_http_connect(*([404] * 100)) # nonexistent: all backends 404
- req = Request.blank('/v1/a', headers={"Accept": "mystery/meat"},
- environ={'REQUEST_METHOD': 'GET',
- 'PATH_INFO': '/v1/a'})
- resp = req.get_response(self.app)
- self.assertEqual(406, resp.status_int)
-
- def test_GET_autocreate_format_invalid_utf8(self):
- with save_globals():
- set_http_connect(*([404] * 100)) # nonexistent: all backends 404
- req = Request.blank('/v1/a?format=\xff\xfe',
- environ={'REQUEST_METHOD': 'GET',
- 'PATH_INFO': '/v1/a',
- 'QUERY_STRING': 'format=\xff\xfe'})
- resp = req.get_response(self.app)
- self.assertEqual(400, resp.status_int)
-
- def test_account_acl_header_access(self):
- acl = {
- 'admin': ['AUTH_alice'],
- 'read-write': ['AUTH_bob'],
- 'read-only': ['AUTH_carol'],
- }
- prefix = get_sys_meta_prefix('account')
- privileged_headers = {(prefix + 'core-access-control'): format_acl(
- version=2, acl_dict=acl)}
-
- app = proxy_server.Application(
- None, FakeMemcache(), account_ring=FakeRing(),
- container_ring=FakeRing())
-
- with save_globals():
- # Mock account server will provide privileged information (ACLs)
- set_http_connect(200, 200, 200, headers=privileged_headers)
- req = Request.blank('/v1/a', environ={'REQUEST_METHOD': 'GET'})
- resp = app.handle_request(req)
-
- # Not a swift_owner -- ACLs should NOT be in response
- header = 'X-Account-Access-Control'
- self.assertNotIn(header, resp.headers, '%r was in %r' % (
- header, resp.headers))
-
- # Same setup -- mock acct server will provide ACLs
- set_http_connect(200, 200, 200, headers=privileged_headers)
- req = Request.blank('/v1/a', environ={'REQUEST_METHOD': 'GET',
- 'swift_owner': True})
- resp = app.handle_request(req)
-
- # For a swift_owner, the ACLs *should* be in response
- self.assertIn(header, resp.headers, '%r not in %r' % (
- header, resp.headers))
-
- def test_account_acls_through_delegation(self):
-
- # Define a way to grab the requests sent out from the AccountController
- # to the Account Server, and a way to inject responses we'd like the
- # Account Server to return.
- resps_to_send = []
-
- @contextmanager
- def patch_account_controller_method(verb):
- old_method = getattr(proxy_server.AccountController, verb)
- new_method = lambda self, req, *_, **__: resps_to_send.pop(0)
- try:
- setattr(proxy_server.AccountController, verb, new_method)
- yield
- finally:
- setattr(proxy_server.AccountController, verb, old_method)
-
- def make_test_request(http_method, swift_owner=True):
- env = {
- 'REQUEST_METHOD': http_method,
- 'swift_owner': swift_owner,
- }
- acl = {
- 'admin': ['foo'],
- 'read-write': ['bar'],
- 'read-only': ['bas'],
- }
- headers = {} if http_method in ('GET', 'HEAD') else {
- 'x-account-access-control': format_acl(version=2, acl_dict=acl)
- }
-
- return Request.blank('/v1/a', environ=env, headers=headers)
-
- # Our AccountController will invoke methods to communicate with the
- # Account Server, and they will return responses like these:
- def make_canned_response(http_method):
- acl = {
- 'admin': ['foo'],
- 'read-write': ['bar'],
- 'read-only': ['bas'],
- }
- headers = {'x-account-sysmeta-core-access-control': format_acl(
- version=2, acl_dict=acl)}
- canned_resp = Response(headers=headers)
- canned_resp.environ = {
- 'PATH_INFO': '/acct',
- 'REQUEST_METHOD': http_method,
- }
- resps_to_send.append(canned_resp)
-
- app = proxy_server.Application(
- None, FakeMemcache(), account_ring=FakeRing(),
- container_ring=FakeRing())
- app.allow_account_management = True
-
- ext_header = 'x-account-access-control'
- with patch_account_controller_method('GETorHEAD_base'):
- # GET/HEAD requests should remap sysmeta headers from acct server
- for verb in ('GET', 'HEAD'):
- make_canned_response(verb)
- req = make_test_request(verb)
- resp = app.handle_request(req)
- h = parse_acl(version=2, data=resp.headers.get(ext_header))
- self.assertEqual(h['admin'], ['foo'])
- self.assertEqual(h['read-write'], ['bar'])
- self.assertEqual(h['read-only'], ['bas'])
-
- # swift_owner = False: GET/HEAD shouldn't return sensitive info
- make_canned_response(verb)
- req = make_test_request(verb, swift_owner=False)
- resp = app.handle_request(req)
- h = resp.headers
- self.assertIsNone(h.get(ext_header))
-
- # swift_owner unset: GET/HEAD shouldn't return sensitive info
- make_canned_response(verb)
- req = make_test_request(verb, swift_owner=False)
- del req.environ['swift_owner']
- resp = app.handle_request(req)
- h = resp.headers
- self.assertIsNone(h.get(ext_header))
-
- # Verify that PUT/POST requests remap sysmeta headers from acct server
- with patch_account_controller_method('make_requests'):
- make_canned_response('PUT')
- req = make_test_request('PUT')
- resp = app.handle_request(req)
-
- h = parse_acl(version=2, data=resp.headers.get(ext_header))
- self.assertEqual(h['admin'], ['foo'])
- self.assertEqual(h['read-write'], ['bar'])
- self.assertEqual(h['read-only'], ['bas'])
-
- make_canned_response('POST')
- req = make_test_request('POST')
- resp = app.handle_request(req)
-
- h = parse_acl(version=2, data=resp.headers.get(ext_header))
- self.assertEqual(h['admin'], ['foo'])
- self.assertEqual(h['read-write'], ['bar'])
- self.assertEqual(h['read-only'], ['bas'])
-
-
class FakeObjectController(object):
def __init__(self):
--
2.7.4
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment