werst_preaube_evar.patch

diff --git a/test/probe/test_reconstructor_rebuild.py b/test/probe/test_reconstructor_rebuild.py
index 844c539..19cefbf 100644
--- a/test/probe/test_reconstructor_rebuild.py
+++ b/test/probe/test_reconstructor_rebuild.py
@@ -14,12 +14,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import errno
 from hashlib import md5
 import unittest
 import uuid
 import shutil
 import random
 from collections import defaultdict
+import os
 
 from test.probe.common import ECProbeTest
 
@@ -222,6 +224,84 @@ class TestReconstructorRebuild(ECProbeTest):
         # just to be nice
         self.revive_drive(device_path)
 
+    def test_rebuild_with_multiple_failures(self):
+        # create EC container
+        headers = {'X-Storage-Policy': self.policy.name}
+        client.put_container(self.url, self.token, self.container_name,
+                             headers=headers)
+
+        # PUT object
+        contents = Body()
+        etag = client.put_object(self.url, self.token,
+                                 self.container_name,
+                                 self.object_name,
+                                 contents=contents)
+
+        opart, onodes = self.object_ring.get_nodes(
+            self.account, self.container_name, self.object_name)
+
+        # pick a couple of nodes at random
+        failed_nodes = random.sample(onodes, 2)
+        # ... and delete those parts
+        for node in failed_nodes:
+            part_dir = self.storage_dir('object', node, part=opart)
+            shutil.rmtree(part_dir, True)
+
+        self.reconstructor.once()
+
+        # we can get it form the proxy
+        self.assertEqual(etag, self.proxy_get())
+        # ... and all frags are in place
+        for node in onodes:
+            try:
+                self.direct_get(node, opart)
+            except direct_client.DirectClientException as err:
+                self.fail('Node %r raised %r' % (node, err))
+
+        scenario_indexes = [
+            (0, 2, 4),  # werx
+            (0, 1, 2),  # werx
+            (0, 4, 2),  # werx
+            (0, 2, 1),  # ... you're kidding me ...
+        ]
+        for failure_indexes in scenario_indexes:
+            # this time we'll pick three nodes
+            failed_nodes = [onodes[i] for i in failure_indexes]
+            self._break_nodes(opart, failed_nodes)
+
+            for i in range(3):
+                self.reconstructor.once()
+
+            # we can get it form the proxy
+            self.assertEqual(etag, self.proxy_get())
+            # ... and all frags are in place
+            for node in onodes:
+                try:
+                    self.direct_get(node, opart)
+                except direct_client.DirectClientException as err:
+                    self.fail('Node %r raised %r for failure_indexes %r' % (
+                        node, err, failure_indexes))
+
+    def _break_nodes(self, opart, failed_nodes):
+        # ... and delete parts on the first *two*
+        for node in failed_nodes[:2]:
+            part_dir = self.storage_dir('object', node, part=opart)
+            shutil.rmtree(part_dir, True)
+        # ... and the durable on the last one
+        for node in failed_nodes[-1:]:
+            part_dir = self.storage_dir('object', node, part=opart)
+            for dirs, subdirs, files in os.walk(part_dir):
+                for fname in files:
+                    if fname.endswith('.durable'):
+                        durable = os.path.join(dirs, fname)
+                        os.remove(durable)
+                        break
+            try:
+                os.remove(os.path.join(part_dir, 'hashes.pkl'))
+            except OSError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+
 
 if __name__ == "__main__":
     unittest.main()

I wonder if it could be worth to understand if the scenario could consist of some thing like

scenarios = [{'failures': (0, 2), 'non_durable': (4,)}, ...]

rather than simple 3 item tuples. I know L87 describes obviously though.