Skip to content

Instantly share code, notes, and snippets.

@alistairncoles
Created March 17, 2017 18:40
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save alistairncoles/ccaf621c64e10224911328c62f62b0f0 to your computer and use it in GitHub Desktop.
Save alistairncoles/ccaf621c64e10224911328c62f62b0f0 to your computer and use it in GitHub Desktop.
diff --git a/swift/common/ring/composite_builder.py b/swift/common/ring/composite_builder.py
index 27afc55e0..f73225a9d 100644
--- a/swift/common/ring/composite_builder.py
+++ b/swift/common/ring/composite_builder.py
@@ -55,8 +55,9 @@ def pre_validate_all_builders(builders):
# check regions
regions_info = {}
- for i, builder in enumerate(builders):
- regions_info[i] = set([dev['region'] for dev in builder.devs])
+ for builder in builders:
+ regions_info[builder] = set(
+ [dev['region'] for dev in builder._iter_devs()])
for first, second in combinations(regions_info.items(), 2):
inter = first[1] & (second[1])
if inter:
@@ -68,23 +69,26 @@ def pre_validate_all_builders(builders):
raise ValueError('Same region found in different rings')
-def validate_dev_uniqueness(ring_data):
- """
- Post-validation for the new zipped rings contains no duplicate devices
+def check_for_dev_uniqueness(builders):
+ builder2devs = {}
- :param ring_data: a RingData insatnce for the zipped ring
- """
- # To be simple, just the validate zipped ring here, it is impossible
- # to report which ring files the incorrect devices appeared in though.
- ip_port_dev_set = set()
- for dev in ring_data.devs:
- if (dev['ip'], dev['port'], dev['device']) in ip_port_dev_set:
- raise ValueError(
- 'Duplicate ip/port/device combination found '
- 'in the zipped ring file, %s/%s/%s' %
- (dev['ip'], dev['port'], dev['device']))
- else:
- ip_port_dev_set.add((dev['ip'], dev['port'], dev['device']))
+ def check_dev(ip, port, dev):
+ for other_builder, devs in builder2devs.items():
+ if (ip, port, dev) in devs:
+ raise ValueError(
+ 'Duplicate ip/port/device combination %s/%s/%s found '
+ 'in builders %s and %s' %
+ (ip, port, dev, other_builder.builder_file,
+ builder.builder_file)
+ )
+
+ for builder in builders:
+ dev_set = set()
+ for dev in builder._iter_devs():
+ dev_params = (dev['ip'], dev['port'], dev['device'])
+ check_dev(*dev_params)
+ dev_set.add(dev_params)
+ builder2devs[builder] = dev_set
def composite(ring_builders):
@@ -96,6 +100,7 @@ def composite(ring_builders):
"""
pre_validate_all_builders(ring_builders)
+ check_for_dev_uniqueness(ring_builders)
zipped_r2p2d = []
zipped_devs = []
composite_info = []
@@ -120,5 +125,4 @@ def composite(ring_builders):
rd = RingData(zipped_r2p2d, zipped_devs, ring_builders[0].part_shift,
{'composite_info': composite_info})
- validate_dev_uniqueness(rd)
return rd
diff --git a/test/unit/common/ring/test_composite_builder.py b/test/unit/common/ring/test_composite_builder.py
index 63a659876..668d05ab1 100644
--- a/test/unit/common/ring/test_composite_builder.py
+++ b/test/unit/common/ring/test_composite_builder.py
@@ -68,9 +68,11 @@ class TestCompositeRingBuilder(unittest.TestCase):
for region in range(num_builders):
_, fname = tempfile.mkstemp(dir=self.tmpdir)
builder = RingBuilder(6, 3, 1, fname)
- for _ in range(4):
+ for _ in range(5):
dev = self.pop_region_device(region)
builder.add_dev(dev)
+ # remove last dev to simulate a ring with some history
+ builder.remove_dev(dev['id'])
builder.rebalance()
# TODO: make sure if we need to save the builder info as
# persistent
@@ -90,11 +92,11 @@ class TestCompositeRingBuilder(unittest.TestCase):
for builder in builders:
builder_devs.extend([
(dev['ip'], dev['port'], dev['device'])
- for dev in builder.devs])
+ for dev in builder._iter_devs()])
got_devices = [
(dev['ip'], dev['port'], dev['device'])
- for dev in zipped_ring.devs]
+ for dev in zipped_ring.devs if dev]
self.assertEqual(sorted(builder_devs), sorted(got_devices),
"zipped_ring mismatched with part of the rings")
@@ -214,8 +216,8 @@ class TestCompositeRingBuilder(unittest.TestCase):
with self.assertRaises(ValueError) as cm:
composite(builders)
self.assertIn(
- 'Duplicate ip/port/device combination found in '
- 'the zipped ring file', cm.exception.message)
+ 'Duplicate ip/port/device combination %(ip)s/%(port)s/%(device)s '
+ 'found in builders ' % same_device, cm.exception.message)
def test_different_part_power_error(self):
# create a ring builder
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment