fix_porbe_nondefault

From 25fd26acb26dbfec959daabca4e16350eed3f83b Mon Sep 17 00:00:00 2001
From: Yuan Zhou <yuan.zhou@intel.com>
Date: Tue, 15 Apr 2014 16:01:34 +0800
Subject: [PATCH] Fixes probe tests with non-default storage policy

Adding Storage Policy Index to direct client headers, use default
policy if none specified

Raise SkipTest if default object replica count not equals to 3 in
handoff related probe tests

Change-Id: I55e76828a5b26a1d87f8755fc5c25707b6a3e6fc
---
 swift/common/direct_client.py                  |    3 +++
 test/probe/common.py                           |    3 ++-
 test/probe/test_empty_device_handoff.py        |   12 +++++++++---
 test/probe/test_object_failures.py             |    9 +++++----
 test/probe/test_object_handoff.py              |    5 ++++-
 test/probe/test_replication_servers_working.py |   23 +++++++++++++----------
 test/unit/common/test_direct_client.py         |   12 ++++++++----
 7 files changed, 44 insertions(+), 23 deletions(-)

diff --git a/swift/common/direct_client.py b/swift/common/direct_client.py
index 9ac2f8b..494958d 100644
--- a/swift/common/direct_client.py
+++ b/swift/common/direct_client.py
@@ -32,6 +32,7 @@ from swift.common.http import HTTP_NO_CONTENT, HTTP_INSUFFICIENT_STORAGE, \
     is_success, is_server_error
 from swift.common.swob import HeaderKeyDict
 from swift.common.utils import quote
+from swift.common.storage_policy import POLICIES, POLICY_INDEX
 
 try:
     import simplejson as json
@@ -93,6 +94,8 @@ def gen_headers(hdrs_in=None, add_ts=False):
     if add_ts:
         hdrs_out['X-Timestamp'] = normalize_timestamp(time())
     hdrs_out['User-Agent'] = 'direct-client %s' % os.getpid()
+    if POLICY_INDEX not in hdrs_out:
+        hdrs_out[POLICY_INDEX] = str(POLICIES.default.idx)
     return hdrs_out
 
 
diff --git a/test/probe/common.py b/test/probe/common.py
index 3c9192f..c9f529b 100644
--- a/test/probe/common.py
+++ b/test/probe/common.py
@@ -25,6 +25,7 @@ from swiftclient import get_auth, head_account
 from swift.common.ring import Ring
 from swift.common.utils import readconf
 from swift.common.manager import Manager
+from swift.common.storage_policy import POLICIES
 
 from test.probe import CHECK_SERVER_TIMEOUT, VALIDATE_RSYNC
 
@@ -197,7 +198,7 @@ def reset_environment():
     try:
         account_ring = get_ring('account')
         container_ring = get_ring('container')
-        object_ring = get_ring('object')
+        object_ring = get_ring(POLICIES.default.ring_name)
         Manager(['main']).start(wait=False)
         port2server = {}
         for server, port in [('account', 6002), ('container', 6001),
diff --git a/test/probe/test_empty_device_handoff.py b/test/probe/test_empty_device_handoff.py
index 2841bcd..9bcd3ac 100755
--- a/test/probe/test_empty_device_handoff.py
+++ b/test/probe/test_empty_device_handoff.py
@@ -18,12 +18,14 @@ import os
 import shutil
 import time
 
-from unittest import main, TestCase
+from unittest import main, TestCase, SkipTest
 from uuid import uuid4
 
 from swiftclient import client
 
 from swift.common import direct_client
+from swift.common.storage_policy import POLICIES
+from swift.obj.diskfile import get_data_dir
 from swift.common.exceptions import ClientException
 from test.probe.common import kill_server, kill_servers, reset_environment,\
     start_server
@@ -37,6 +39,9 @@ class TestEmptyDevice(TestCase):
         (self.pids, self.port2server, self.account_ring, self.container_ring,
          self.object_ring, self.url, self.token,
          self.account, self.configs) = reset_environment()
+        if self.object_ring.replica_count != 3:
+            raise SkipTest('Default Ring replica is %s instead of 3' %
+                           self.object_ring.replica_count)
 
     def tearDown(self):
         kill_servers(self.port2server, self.pids)
@@ -52,7 +57,7 @@ class TestEmptyDevice(TestCase):
     def test_main(self):
         # Create container
         # Kill one container/obj primary server
-        # Delete the "objects" directory on the primary server
+        # Delete the default data dir directory on the primary server
         # Create container/obj (goes to two primary servers and one handoff)
         # Kill other two container/obj primary servers
         # Indirectly through proxy assert we can get container/obj
@@ -76,7 +81,8 @@ class TestEmptyDevice(TestCase):
             self.account, container, obj)
         onode = onodes[0]
         kill_server(onode['port'], self.port2server, self.pids)
-        obj_dir = '%s/objects' % self._get_objects_dir(onode)
+        obj_dir = '%s/%s' % (self._get_objects_dir(onode),
+                             get_data_dir(POLICIES.default.idx))
         shutil.rmtree(obj_dir, True)
         self.assertFalse(os.path.exists(obj_dir))
         client.put_object(self.url, self.token, container, obj, 'VERIFY')
diff --git a/test/probe/test_object_failures.py b/test/probe/test_object_failures.py
index f8fc119..3799fab 100755
--- a/test/probe/test_object_failures.py
+++ b/test/probe/test_object_failures.py
@@ -23,9 +23,10 @@ from uuid import uuid4
 from swiftclient import client
 
 from swift.common import direct_client
+from swift.common.storage_policy import POLICIES
 from swift.common.exceptions import ClientException
 from swift.common.utils import hash_path, readconf
-from swift.obj.diskfile import write_metadata, read_metadata
+from swift.obj.diskfile import write_metadata, read_metadata, get_data_dir
 from test.probe.common import kill_servers, reset_environment
 
 
@@ -72,9 +73,9 @@ class TestObjectFailures(TestCase):
         hash_str = hash_path(self.account, container, obj)
         obj_server_conf = readconf(self.configs['object-server'][node_id])
         devices = obj_server_conf['app:object-server']['devices']
-        obj_dir = '%s/%s/objects/%s/%s/%s/' % (devices,
-                                               device, opart,
-                                               hash_str[-3:], hash_str)
+        obj_dir = '%s/%s/%s/%s/%s/%s/' % (devices, device,
+                                          get_data_dir(POLICIES.default.idx),
+                                          opart, hash_str[-3:], hash_str)
         data_file = get_data_file_path(obj_dir)
         return onode, opart, data_file
 
diff --git a/test/probe/test_object_handoff.py b/test/probe/test_object_handoff.py
index 565b2ff..4342611 100755
--- a/test/probe/test_object_handoff.py
+++ b/test/probe/test_object_handoff.py
@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from unittest import main, TestCase
+from unittest import main, TestCase, SkipTest
 from uuid import uuid4
 
 from swiftclient import client
@@ -32,6 +32,9 @@ class TestObjectHandoff(TestCase):
         (self.pids, self.port2server, self.account_ring, self.container_ring,
          self.object_ring, self.url, self.token,
          self.account, self.configs) = reset_environment()
+        if self.object_ring.replica_count != 3:
+            raise SkipTest('Default Ring replica is %s instead of 3' %
+                           self.object_ring.replica_count)
 
     def tearDown(self):
         kill_servers(self.port2server, self.pids)
diff --git a/test/probe/test_replication_servers_working.py b/test/probe/test_replication_servers_working.py
index dae315a..c774fdc 100644
--- a/test/probe/test_replication_servers_working.py
+++ b/test/probe/test_replication_servers_working.py
@@ -14,13 +14,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from unittest import main, TestCase
+from unittest import main, TestCase, SkipTest
 from uuid import uuid4
 import os
 import time
 import shutil
 
 from swiftclient import client
+from swift.common.storage_policy import POLICIES
+from swift.obj.diskfile import get_data_dir
 
 from test.probe.common import kill_servers, reset_environment
 from swift.common.utils import readconf
@@ -100,6 +102,7 @@ class TestReplicatorFunctions(TestCase):
         # Delete file "hashes.pkl".
         # Check, that all files were replicated.
         path_list = []
+        data_dir = get_data_dir(POLICIES.default.idx)
         # Figure out where the devices are
         for node_id in range(1, 5):
             conf = readconf(self.configs['object-server'][node_id])
@@ -155,24 +158,24 @@ class TestReplicatorFunctions(TestCase):
                     time.sleep(1)
 
             # Check behavior by deleting hashes.pkl file
-            for directory in os.listdir(os.path.join(test_node, 'objects')):
+            for directory in os.listdir(os.path.join(test_node, data_dir)):
                 for input_dir in os.listdir(os.path.join(
-                        test_node, 'objects', directory)):
+                        test_node, data_dir, directory)):
                     if os.path.isdir(os.path.join(
-                            test_node, 'objects', directory, input_dir)):
+                            test_node, data_dir, directory, input_dir)):
                         shutil.rmtree(os.path.join(
-                            test_node, 'objects', directory, input_dir))
+                            test_node, data_dir, directory, input_dir))
 
             # We will keep trying these tests until they pass for up to 60s
             begin = time.time()
             while True:
                 try:
                     for directory in os.listdir(os.path.join(
-                            test_node, 'objects')):
+                            test_node, data_dir)):
                         for input_dir in os.listdir(os.path.join(
-                                test_node, 'objects', directory)):
+                                test_node, data_dir, directory)):
                             self.assertFalse(os.path.isdir(
-                                os.path.join(test_node, 'objects',
+                                os.path.join(test_node, data_dir,
                                              directory, '/', input_dir)))
                     break
                 except Exception:
@@ -180,9 +183,9 @@ class TestReplicatorFunctions(TestCase):
                         raise
                     time.sleep(1)
 
-            for directory in os.listdir(os.path.join(test_node, 'objects')):
+            for directory in os.listdir(os.path.join(test_node, data_dir)):
                 os.remove(os.path.join(
-                    test_node, 'objects', directory, 'hashes.pkl'))
+                    test_node, data_dir, directory, 'hashes.pkl'))
 
             # We will keep trying these tests until they pass for up to 60s
             begin = time.time()
diff --git a/test/unit/common/test_direct_client.py b/test/unit/common/test_direct_client.py
index 0a85c35..91f497c 100644
--- a/test/unit/common/test_direct_client.py
+++ b/test/unit/common/test_direct_client.py
@@ -28,7 +28,9 @@ from swift.common import direct_client
 from swift.common.exceptions import ClientException
 from swift.common.utils import json, normalize_timestamp
 from swift.common.swob import HeaderKeyDict, RESPONSE_REASONS
+from swift.common.storage_policy import POLICY_INDEX
 
+from test.unit import patch_policies
 
 class FakeConn(object):
 
@@ -82,6 +84,7 @@ def mocked_http_conn(*args, **kwargs):
         yield fake_conn
 
 
+@patch_policies
 class TestDirectClient(unittest.TestCase):
 
     def setUp(self):
@@ -106,23 +109,24 @@ class TestDirectClient(unittest.TestCase):
         hdrs = direct_client.gen_headers()
         assert 'user-agent' in hdrs
         assert hdrs['user-agent'] == 'direct-client %s' % os.getpid()
-        assert len(hdrs.keys()) == 1
+        assert hdrs[POLICY_INDEX] == '0'
+        assert len(hdrs.keys()) == 2
 
         hdrs = direct_client.gen_headers(add_ts=True)
         assert 'user-agent' in hdrs
         assert 'x-timestamp' in hdrs
-        assert len(hdrs.keys()) == 2
+        assert len(hdrs.keys()) == 3
 
         hdrs = direct_client.gen_headers(hdrs_in={'foo-bar': '47'})
         assert 'user-agent' in hdrs
         assert 'foo-bar' in hdrs
         assert hdrs['foo-bar'] == '47'
-        assert len(hdrs.keys()) == 2
+        assert len(hdrs.keys()) == 3
 
         hdrs = direct_client.gen_headers(hdrs_in={'user-agent': '47'})
         assert 'user-agent' in hdrs
         assert hdrs['user-agent'] == 'direct-client %s' % os.getpid()
-        assert len(hdrs.keys()) == 1
+        assert len(hdrs.keys()) == 2
 
     def test_direct_get_account(self):
         stub_headers = {
-- 
1.7.9.5