Skip to content

Instantly share code, notes, and snippets.

@notmyname

notmyname/gist:9fd0a28fa36b9fe9c1de Secret

Created April 14, 2015 03:12
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save notmyname/9fd0a28fa36b9fe9c1de to your computer and use it in GitHub Desktop.
Save notmyname/9fd0a28fa36b9fe9c1de to your computer and use it in GitHub Desktop.
Download ZIP
Raw
gistfile1.txt
From 2b87d7e6033484c8c3dcd25bd326a3355a7c886a Mon Sep 17 00:00:00 2001
From: John Dickinson <me@not.mn>
Date: Mon, 13 Apr 2015 19:34:33 -0700
Subject: [PATCH] fragfragfrag
Change-Id: I81e617f737941d9588f5847e24f78719ddf1b5ea
---
doc/source/overview_erasure_code.rst | 13 ++++++------
swift/obj/diskfile.py | 2 +-
swift/obj/reconstructor.py | 2 +-
swift/proxy/controllers/obj.py | 2 +-
test/unit/obj/test_diskfile.py | 38 ++++++++++++++++++------------------
test/unit/obj/test_reconstructor.py | 4 ++--
test/unit/obj/test_server.py | 24 +++++++++++------------
test/unit/obj/test_ssync_sender.py | 8 ++++----
test/unit/proxy/test_server.py | 2 +-
9 files changed, 48 insertions(+), 47 deletions(-)
diff --git a/doc/source/overview_erasure_code.rst b/doc/source/overview_erasure_code.rst
index cd16ae1..7915ae2 100755
--- a/doc/source/overview_erasure_code.rst
+++ b/doc/source/overview_erasure_code.rst
@@ -516,21 +516,22 @@ Metadata
There are few different categories of metadata that are associated with EC:
-System Metadata: EC has a set of object level system metadata that it attaches
-to each of the EC archives. The metadata is for internal use only:
+System Metadata: EC has a set of object level system metadata that it
+attaches to each of the EC fragment archives. The metadata is for
+internal use only:
* ``X-Object-Sysmeta-EC-Etag``: The Etag of the original object.
* ``X-Object-Sysmeta-EC-Content-Length``: The content length of the original
object.
-* ``X-Object-Sysmeta-EC-Archive-Index``: Also known as the "fragment index".
+* ``X-Object-Sysmeta-Ec-Frag-Index``: The index for this fragment.
* ``X-Object-Sysmeta-EC-Scheme``: Description of the EC policy used to encode
the object.
* ``X-Object-Sysmeta-EC-Segment-Size``: The segment size used for the object.
User Metadata: User metadata is unaffected by EC, however, a full copy of the
-user metadata is stored with every EC archive. This is required as the
-reconstructor needs this information and each reconstructor only communicates
-with its closest neighbors on the ring.
+user metadata is stored with every EC fragment archive. This is required as
+the reconstructor needs this information and each reconstructor only
+communicates with its closest neighbors on the ring.
PyECLib Metadata: PyECLib stores a small amount of metadata on a per fragment
basis. This metadata is not documented here as it is opaque to Swift.
diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py
index e6f25ee..88d6931 100644
--- a/swift/obj/diskfile.py
+++ b/swift/obj/diskfile.py
@@ -1837,7 +1837,7 @@ class ECDiskFileWriter(DiskFileWriter):
# canon, but if it's unavailable (e.g. tests) it's reasonable to
# use the frag_index provided at instantiation. Either way make
# sure that the fragment index is included in object sysmeta.
- fi = metadata.setdefault('X-Object-Sysmeta-Ec-Archive-Index',
+ fi = metadata.setdefault('X-Object-Sysmeta-Ec-Frag-Index',
self._diskfile._frag_index)
filename = self.manager.make_on_disk_filename(
timestamp, self._extension, frag_index=fi)
diff --git a/swift/obj/reconstructor.py b/swift/obj/reconstructor.py
index e4ca6b0..92bf462 100644
--- a/swift/obj/reconstructor.py
+++ b/swift/obj/reconstructor.py
@@ -64,7 +64,7 @@ class RebuildingECDiskFileStream(object):
# update the FI and delete the ETag, the obj server will
# recalc on the other side...
- self.metadata['X-Object-Sysmeta-Ec-Archive-Index'] = frag_index
+ self.metadata['X-Object-Sysmeta-Ec-Frag-Index'] = frag_index
del self.metadata['ETag']
self.frag_index = frag_index
diff --git a/swift/proxy/controllers/obj.py b/swift/proxy/controllers/obj.py
index d31d05e..04f8271 100644
--- a/swift/proxy/controllers/obj.py
+++ b/swift/proxy/controllers/obj.py
@@ -1736,7 +1736,7 @@ def trailing_metadata(policy, client_obj_hasher,
client_obj_hasher.hexdigest(),
'X-Backend-Container-Update-Override-Size':
str(bytes_transferred_from_client),
- 'X-Object-Sysmeta-EC-Archive-Index': str(fragment_archive_index),
+ 'X-Object-Sysmeta-Ec-Frag-Index': str(fragment_archive_index),
# These fields are for debuggability,
# AKA "what is this thing?"
'X-Object-Sysmeta-EC-Scheme': policy.ec_scheme_description,
diff --git a/test/unit/obj/test_diskfile.py b/test/unit/obj/test_diskfile.py
index 1c51a59..6b64a1c 100644
--- a/test/unit/obj/test_diskfile.py
+++ b/test/unit/obj/test_diskfile.py
@@ -1734,7 +1734,7 @@ class TestECDiskFileManager(DiskFileManagerMixin, unittest.TestCase):
'ETag': md5(data).hexdigest(),
'X-Timestamp': timestamp.internal,
'Content-Length': len(data),
- 'X-Object-Sysmeta-Ec-Archive-Index': str(frag_index),
+ 'X-Object-Sysmeta-Ec-Frag-Index': str(frag_index),
}
writer.put(metadata)
writer.commit(timestamp)
@@ -1743,12 +1743,12 @@ class TestECDiskFileManager(DiskFileManagerMixin, unittest.TestCase):
self.existing_device1, '0', hash_, POLICIES.default, frag_index=4)
self.assertEqual(df4._frag_index, 4)
self.assertEqual(
- df4.read_metadata()['X-Object-Sysmeta-Ec-Archive-Index'], '4')
+ df4.read_metadata()['X-Object-Sysmeta-Ec-Frag-Index'], '4')
df7 = self.df_mgr.get_diskfile_from_hash(
self.existing_device1, '0', hash_, POLICIES.default, frag_index=7)
self.assertEqual(df7._frag_index, 7)
self.assertEqual(
- df7.read_metadata()['X-Object-Sysmeta-Ec-Archive-Index'], '7')
+ df7.read_metadata()['X-Object-Sysmeta-Ec-Frag-Index'], '7')
class DiskFileMixin(BaseDiskFileTestMixin):
@@ -2214,7 +2214,7 @@ class DiskFileMixin(BaseDiskFileTestMixin):
self.df_router = diskfile.DiskFileRouter(self.conf, self.logger)
# actual on disk frag_index may have been set by metadata
- frag_index = metadata.get('X-Object-Sysmeta-Ec-Archive-Index',
+ frag_index = metadata.get('X-Object-Sysmeta-Ec-Frag-Index',
frag_index)
df = self._simple_get_diskfile(obj=obj_name, policy=policy,
frag_index=frag_index)
@@ -2587,7 +2587,7 @@ class DiskFileMixin(BaseDiskFileTestMixin):
'Content-Length': '0',
}
if policy.policy_type == EC_POLICY:
- metadata['X-Object-Sysmeta-Ec-Archive-Index'] = \
+ metadata['X-Object-Sysmeta-Ec-Frag-Index'] = \
df._frag_index or 7
writer.put(metadata)
writer.commit(timestamp)
@@ -2686,7 +2686,7 @@ class DiskFileMixin(BaseDiskFileTestMixin):
def test_delete(self):
for policy in POLICIES:
if policy.policy_type == EC_POLICY:
- metadata = {'X-Object-Sysmeta-Ec-Archive-Index': '1'}
+ metadata = {'X-Object-Sysmeta-Ec-Frag-Index': '1'}
fi = 1
else:
metadata = {}
@@ -3271,33 +3271,33 @@ class TestECDiskFile(DiskFileMixin, unittest.TestCase):
frag_index=good_value)
self.assertEqual(expected, sorted(os.listdir(df._datadir)))
# frag index should be added to object sysmeta
- actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Archive-Index')
+ actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Frag-Index')
self.assertEqual(int(good_value), int(actual))
# metadata value overrides the constructor arg
ts = self.ts().internal
expected = ['%s#%s.data' % (ts, good_value), '%s.durable' % ts]
- meta = {'X-Object-Sysmeta-Ec-Archive-Index': good_value}
+ meta = {'X-Object-Sysmeta-Ec-Frag-Index': good_value}
df = self._get_open_disk_file(ts=ts, policy=policy,
frag_index='99',
extra_metadata=meta)
self.assertEqual(expected, sorted(os.listdir(df._datadir)))
- actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Archive-Index')
+ actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Frag-Index')
self.assertEqual(int(good_value), int(actual))
# metadata value alone is sufficient
ts = self.ts().internal
expected = ['%s#%s.data' % (ts, good_value), '%s.durable' % ts]
- meta = {'X-Object-Sysmeta-Ec-Archive-Index': good_value}
+ meta = {'X-Object-Sysmeta-Ec-Frag-Index': good_value}
df = self._get_open_disk_file(ts=ts, policy=policy,
frag_index=None,
extra_metadata=meta)
self.assertEqual(expected, sorted(os.listdir(df._datadir)))
- actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Archive-Index')
+ actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Frag-Index')
self.assertEqual(int(good_value), int(actual))
def test_sysmeta_frag_index_is_immutable(self):
- # the X-Object-Sysmeta-Ec-Archive-Index should *only* be set when
+ # the X-Object-Sysmeta-Ec-Frag-Index should *only* be set when
# the .data file is written.
policy = POLICIES.default
orig_frag_index = 14
@@ -3308,7 +3308,7 @@ class TestECDiskFile(DiskFileMixin, unittest.TestCase):
frag_index=orig_frag_index)
self.assertEqual(expected, sorted(os.listdir(df._datadir)))
# frag index should be added to object sysmeta
- actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Archive-Index')
+ actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Frag-Index')
self.assertEqual(int(orig_frag_index), int(actual))
# open the same diskfile with no frag_index passed to constructor
@@ -3316,7 +3316,7 @@ class TestECDiskFile(DiskFileMixin, unittest.TestCase):
self.existing_device, 0, 'a', 'c', 'my_obj', policy=policy,
frag_index=None)
df.open()
- actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Archive-Index')
+ actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Frag-Index')
self.assertEqual(int(orig_frag_index), int(actual))
# write metadata to a meta file
@@ -3338,13 +3338,13 @@ class TestECDiskFile(DiskFileMixin, unittest.TestCase):
self.assertEqual(ts, df.get_metadata().get('X-Timestamp'))
self.assertEqual('kiwi', df.get_metadata().get('X-Object-Meta-Fruit'))
# check frag index sysmeta is unchanged
- actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Archive-Index')
+ actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Frag-Index')
self.assertEqual(int(orig_frag_index), int(actual))
# attempt to overwrite frag index sysmeta
ts = self.ts().internal
metadata = {'X-Timestamp': ts,
- 'X-Object-Sysmeta-Ec-Archive-Index': 99,
+ 'X-Object-Sysmeta-Ec-Frag-Index': 99,
'X-Object-Meta-Fruit': 'apple'}
df.write_metadata(metadata)
@@ -3356,7 +3356,7 @@ class TestECDiskFile(DiskFileMixin, unittest.TestCase):
# sanity check we have read the meta file
self.assertEqual(ts, df.get_metadata().get('X-Timestamp'))
self.assertEqual('apple', df.get_metadata().get('X-Object-Meta-Fruit'))
- actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Archive-Index')
+ actual = df.get_metadata().get('X-Object-Sysmeta-Ec-Frag-Index')
self.assertEqual(int(orig_frag_index), int(actual))
def test_data_file_errors_bad_frag_index(self):
@@ -3373,7 +3373,7 @@ class TestECDiskFile(DiskFileMixin, unittest.TestCase):
df = df_mgr.get_diskfile(self.existing_device, '0', 'a', 'c', 'o',
policy=policy, frag_index=None)
ts = self.ts()
- meta = {'X-Object-Sysmeta-Ec-Archive-Index': bad_value,
+ meta = {'X-Object-Sysmeta-Ec-Frag-Index': bad_value,
'X-Timestamp': ts.internal,
'Content-Length': 0,
'Etag': EMPTY_ETAG,
@@ -3390,7 +3390,7 @@ class TestECDiskFile(DiskFileMixin, unittest.TestCase):
df = df_mgr.get_diskfile(self.existing_device, '0', 'a', 'c', 'o',
policy=policy, frag_index=2)
ts = self.ts()
- meta = {'X-Object-Sysmeta-Ec-Archive-Index': bad_value,
+ meta = {'X-Object-Sysmeta-Ec-Frag-Index': bad_value,
'X-Timestamp': ts.internal,
'Content-Length': 0,
'Etag': EMPTY_ETAG,
diff --git a/test/unit/obj/test_reconstructor.py b/test/unit/obj/test_reconstructor.py
index 1775409..8b4a201 100755
--- a/test/unit/obj/test_reconstructor.py
+++ b/test/unit/obj/test_reconstructor.py
@@ -287,7 +287,7 @@ class TestGlobalSetupObjectReconstructor(unittest.TestCase):
'X-Timestamp': timestamp.internal,
'Content-Length': len(test_data),
'Etag': md5(test_data).hexdigest(),
- 'X-Object-Sysmeta-Ec-Archive-Index': frag_index,
+ 'X-Object-Sysmeta-Ec-Frag-Index': frag_index,
}
writer.put(metadata)
writer.commit(timestamp)
@@ -2199,7 +2199,7 @@ class TestObjectReconstructor(unittest.TestCase):
'X-Timestamp': ts.internal,
'Content-Length': len(test_data),
'Etag': md5(test_data).hexdigest(),
- 'X-Object-Sysmeta-Ec-Archive-Index': frag_index,
+ 'X-Object-Sysmeta-Ec-Frag-Index': frag_index,
}
writer.put(metadata)
writer.commit(ts)
diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py
index 267fd7a..52a3434 100755
--- a/test/unit/obj/test_server.py
+++ b/test/unit/obj/test_server.py
@@ -1275,7 +1275,7 @@ class TestObjectController(unittest.TestCase):
'Content-Type': 'application/octet-stream',
'X-Backend-Storage-Policy-Index': int(policy)}
if policy.policy_type == EC_POLICY:
- headers['X-Object-Sysmeta-Ec-Archive-Index'] = '2'
+ headers['X-Object-Sysmeta-Ec-Frag-Index'] = '2'
data_file_tail = '#2.data'
req = Request.blank(
'/sda1/p/a/c/o', environ={'REQUEST_METHOD': 'PUT'},
@@ -3151,7 +3151,7 @@ class TestObjectController(unittest.TestCase):
'X-Delete-At-Device': 'sda1',
'X-Backend-Storage-Policy-Index': int(policy)}
if policy.policy_type == EC_POLICY:
- headers['X-Object-Sysmeta-Ec-Archive-Index'] = '2'
+ headers['X-Object-Sysmeta-Ec-Frag-Index'] = '2'
req = Request.blank(
'/sda1/p/a/c/o', method='PUT', body='', headers=headers)
with mocked_http_conn(
@@ -3464,7 +3464,7 @@ class TestObjectController(unittest.TestCase):
'X-Container-Partition': 'cpartition',
'X-Container-Device': 'cdevice',
'Content-Type': 'text/plain',
- 'X-Object-Sysmeta-Ec-Archive-Index': 0,
+ 'X-Object-Sysmeta-Ec-Frag-Index': 0,
'X-Backend-Storage-Policy-Index': int(policy)}, body='')
given_args = []
@@ -3757,7 +3757,7 @@ class TestObjectController(unittest.TestCase):
'Content-Length': '4',
'Content-Type': 'application/octet-stream',
'X-Backend-Storage-Policy-Index': int(policy),
- 'X-Object-Sysmeta-Ec-Archive-Index': 2})
+ 'X-Object-Sysmeta-Ec-Frag-Index': 2})
req.body = 'TEST'
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 201)
@@ -3828,7 +3828,7 @@ class TestObjectController(unittest.TestCase):
'Content-Length': '4',
'Content-Type': 'application/octet-stream',
'X-Backend-Storage-Policy-Index': int(policy),
- 'X-Object-Sysmeta-Ec-Archive-Index': 4})
+ 'X-Object-Sysmeta-Ec-Frag-Index': 4})
req.body = 'TEST'
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 201)
@@ -3844,7 +3844,7 @@ class TestObjectController(unittest.TestCase):
'Content-Type': 'application/octet-stream',
'X-Delete-At': delete_at_timestamp1,
'X-Backend-Storage-Policy-Index': int(policy),
- 'X-Object-Sysmeta-Ec-Archive-Index': 3})
+ 'X-Object-Sysmeta-Ec-Frag-Index': 3})
req.body = 'TEST'
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 201)
@@ -3867,7 +3867,7 @@ class TestObjectController(unittest.TestCase):
'Content-Type': 'application/octet-stream',
'X-Delete-At': delete_at_timestamp2,
'X-Backend-Storage-Policy-Index': int(policy),
- 'X-Object-Sysmeta-Ec-Archive-Index': 3})
+ 'X-Object-Sysmeta-Ec-Frag-Index': 3})
req.body = 'TEST'
resp = req.get_response(self.object_controller)
self.assertEquals(resp.status_int, 201)
@@ -4835,7 +4835,7 @@ class TestObjectController(unittest.TestCase):
'Content-Type': 'application/x-test',
'X-Backend-Storage-Policy-Index': index}
if POLICIES[index].policy_type == EC_POLICY:
- headers['X-Object-Sysmeta-Ec-Archive-Index'] = '2'
+ headers['X-Object-Sysmeta-Ec-Frag-Index'] = '2'
req = Request.blank(
'/sda1/p/a/c/o',
environ={'REQUEST_METHOD': method},
@@ -5017,7 +5017,7 @@ class TestObjectServer(unittest.TestCase):
# file along with .data file in the object data directory
test_data = 'obj data'
footer_meta = {
- "X-Object-Sysmeta-Ec-Archive-Index": "2",
+ "X-Object-Sysmeta-Ec-Frag-Index": "2",
"Etag": md5(test_data).hexdigest(),
}
footer_json = json.dumps(footer_meta)
@@ -5119,7 +5119,7 @@ class TestObjectServer(unittest.TestCase):
# test doesn't have, see `test_multiphase_put_metadata_footer`),
# but the proxy *could* send the frag index in the headers and
# this test verifies that would work.
- 'X-Object-Sysmeta-Ec-Archive-Index': '2',
+ 'X-Object-Sysmeta-Ec-Frag-Index': '2',
'X-Backend-Storage-Policy-Index': '1',
'X-Backend-Obj-Content-Length': len(test_data),
'X-Backend-Obj-Multipart-Mime-Boundary': 'boundary123',
@@ -5198,7 +5198,7 @@ class TestObjectServer(unittest.TestCase):
test_data = 'obj data'
footer_meta = {
- "X-Object-Sysmeta-Ec-Archive-Index": "7",
+ "X-Object-Sysmeta-Ec-Frag-Index": "7",
"Etag": md5(test_data).hexdigest(),
}
footer_json = json.dumps(footer_meta)
@@ -5259,7 +5259,7 @@ class TestObjectServer(unittest.TestCase):
# phase commit confirmation being received corrupt
test_data = 'obj data'
footer_meta = {
- "X-Object-Sysmeta-Ec-Archive-Index": "7",
+ "X-Object-Sysmeta-Ec-Frag-Index": "7",
"Etag": md5(test_data).hexdigest(),
}
footer_json = json.dumps(footer_meta)
diff --git a/test/unit/obj/test_ssync_sender.py b/test/unit/obj/test_ssync_sender.py
index 3e89b7d..42bd610 100644
--- a/test/unit/obj/test_ssync_sender.py
+++ b/test/unit/obj/test_ssync_sender.py
@@ -1343,7 +1343,7 @@ class TestSsync(BaseTestSender):
for frag_index in frag_indexes:
object_data = '/a/c/%s___%s' % (obj_name, frag_index)
if frag_index is not None:
- metadata['X-Object-Sysmeta-Ec-Archive-Index'] = str(frag_index)
+ metadata['X-Object-Sysmeta-Ec-Frag-Index'] = str(frag_index)
df = self._make_open_diskfile(
device=self.device, partition=self.partition, account='a',
container='c', obj=obj_name, body=object_data,
@@ -1385,7 +1385,7 @@ class TestSsync(BaseTestSender):
self.assertFalse(rx_metadata)
if frag_index:
rx_metadata = rx_df.get_metadata()
- fi_key = 'X-Object-Sysmeta-Ec-Archive-Index'
+ fi_key = 'X-Object-Sysmeta-Ec-Frag-Index'
self.assertTrue(fi_key in rx_metadata)
self.assertEqual(frag_index, int(rx_metadata[fi_key]))
@@ -1581,7 +1581,7 @@ class TestSsync(BaseTestSender):
for subreq in results.get('tx_updates'):
if subreq.get('method') == 'PUT':
self.assertTrue(
- 'X-Object-Sysmeta-Ec-Archive-Index: %s' % rx_node_index
+ 'X-Object-Sysmeta-Ec-Frag-Index: %s' % rx_node_index
in subreq.get('headers'))
expected_body = '%s___%s' % (subreq['path'], rx_node_index)
self.assertEqual(expected_body, subreq['body'])
@@ -1681,7 +1681,7 @@ class TestSsync(BaseTestSender):
for subreq in results.get('tx_updates'):
if subreq.get('method') == 'PUT':
self.assertTrue(
- 'X-Object-Sysmeta-Ec-Archive-Index: %s' % rx_node_index
+ 'X-Object-Sysmeta-Ec-Frag-Index: %s' % rx_node_index
in subreq.get('headers'))
expected_body = '%s___%s' % (subreq['path'], rx_node_index)
self.assertEqual(expected_body, subreq['body'])
diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py
index 9f1ae64..969c54d 100644
--- a/test/unit/proxy/test_server.py
+++ b/test/unit/proxy/test_server.py
@@ -1378,7 +1378,7 @@ class TestObjectController(unittest.TestCase):
lmeta = dict((k.lower(), v) for k, v in meta.items())
got_indices.add(
- lmeta['x-object-sysmeta-ec-archive-index'])
+ lmeta['x-object-sysmeta-ec-frag-index'])
self.assertEqual(
lmeta['x-object-sysmeta-ec-etag'],
--
2.2.1
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment