Skip to content

Instantly share code, notes, and snippets.

@alistairncoles
Last active March 23, 2016 16:01
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save alistairncoles/8e7aeaf64780299a512e to your computer and use it in GitHub Desktop.
Save alistairncoles/8e7aeaf64780299a512e to your computer and use it in GitHub Desktop.
Slightly simpler auditor cleanup of unexpected files
diff --git a/swift/common/utils.py b/swift/common/utils.py
index 210dd9f..8397277 100644
--- a/swift/common/utils.py
+++ b/swift/common/utils.py
@@ -2124,18 +2124,19 @@ def unlink_older_than(path, mtime):
:param path: path to remove file from
:param mtime: timestamp of oldest file to keep
"""
- filepaths = map(functools.partial(os.path.join, path), listdir(path))
- return unlink_paths_older_than(filepaths, mtime)
+ return unlink_paths_older_than(path, listdir(path), mtime)
-def unlink_paths_older_than(filepaths, mtime):
+def unlink_paths_older_than(path_to_dir, filenames, mtime):
"""
- Remove any files from the given list that that were
+ Remove any files from the given directory in the given list that that were
last modified before mtime.
- :param filepaths: a list of strings, the full paths of files to check
+ :param path_to_dir: path to directory from which files are to be removed
+ :param filenames: a list of strings, the basenames of files to check
:param mtime: timestamp of oldest file to keep
"""
+ filepaths = map(functools.partial(os.path.join, path_to_dir), filenames)
for fpath in filepaths:
try:
if os.path.getmtime(fpath) < mtime:
diff --git a/swift/obj/auditor.py b/swift/obj/auditor.py
index ef0253a..ce94755 100644
--- a/swift/obj/auditor.py
+++ b/swift/obj/auditor.py
@@ -260,12 +260,10 @@ class AuditorWorker(object):
# _ondisk_info attr is initialized to None and filled in by open
ondisk_info_dict = df._ondisk_info or {}
if 'unexpected' in ondisk_info_dict:
- is_rsync_tempfile = lambda fpath: RE_RSYNC_TEMPFILE.match(
- os.path.basename(fpath))
- rsync_tempfile_paths = filter(is_rsync_tempfile,
+ rsync_tempfile_paths = filter(RE_RSYNC_TEMPFILE.match,
ondisk_info_dict['unexpected'])
mtime = time.time() - self.rsync_tempfile_timeout
- unlink_paths_older_than(rsync_tempfile_paths, mtime)
+ unlink_paths_older_than(df._datadir, rsync_tempfile_paths, mtime)
class ObjectAuditor(Daemon):
diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py
index 7ea2f25..1454324 100644
--- a/swift/obj/diskfile.py
+++ b/swift/obj/diskfile.py
@@ -755,10 +755,9 @@ class BaseDiskFileManager(object):
file_info['filename'] = afile
exts[file_info['ext']].append(file_info)
except DiskFileError as e:
- file_path = os.path.join(datadir or '', afile)
- self.logger.warning('Unexpected file %s: %s',
- file_path, e)
- results.setdefault('unexpected', []).append(file_path)
+ self.logger.warning('Unexpected file %s: %s' %
+ (os.path.join(datadir or '', afile), e))
+ results.setdefault('unexpected', []).append(afile)
for ext in exts:
# For each extension sort files into reverse chronological order.
exts[ext] = sorted(
diff --git a/test/unit/obj/test_auditor.py b/test/unit/obj/test_auditor.py
index 335bf90..9d9fbdc 100644
--- a/test/unit/obj/test_auditor.py
+++ b/test/unit/obj/test_auditor.py
@@ -313,21 +313,35 @@ class TestAuditor(unittest.TestCase):
config_path = os.path.join(self.testdir, 'objserver.conf')
stub_config = """
[DEFAULT]
- reclaim_age = 604800
+ reclaim_age = 604801
[object-auditor]
rsync_tempfile_timeout = auto
"""
with open(config_path, 'w') as f:
f.write(textwrap.dedent(stub_config))
- # the Daemon loader will hand the object-audtior config to the
+ # the Daemon loader will hand the object-auditor config to the
# auditor who will build the workers from it
conf = readconf(config_path, 'object-auditor')
auditor_worker = auditor.AuditorWorker(conf, self.logger,
self.rcache, self.devices)
# if there is no object-replicator section we still have to fall back
# to reclaim age because we can't parse the config for that section!
- self.assertEqual(auditor_worker.rsync_tempfile_timeout,
- 604800)
+ self.assertEqual(auditor_worker.rsync_tempfile_timeout, 604801)
+ stub_config = """
+ [DEFAULT]
+ [object-auditor]
+ rsync_tempfile_timeout = auto
+ """
+ with open(config_path, 'w') as f:
+ f.write(textwrap.dedent(stub_config))
+ # the Daemon loader will hand the object-auditor config to the
+ # auditor who will build the workers from it
+ conf = readconf(config_path, 'object-auditor')
+ auditor_worker = auditor.AuditorWorker(conf, self.logger,
+ self.rcache, self.devices)
+ # if there is no object-replicator section and no reclaim age option we
+ # fall back to one week
+ self.assertEqual(auditor_worker.rsync_tempfile_timeout, 604800)
stub_config = """
[DEFAULT]
reclaim_age = 1209600
@@ -341,7 +355,7 @@ class TestAuditor(unittest.TestCase):
auditor_worker = auditor.AuditorWorker(conf, self.logger,
self.rcache, self.devices)
# if the object-replicator section will parse but does not override
- # the default rsync_timeout we assume the default rsync_tiemout value
+ # the default rsync_timeout we assume the default rsync_timeout value
# and add 15mins
self.assertEqual(auditor_worker.rsync_tempfile_timeout,
replicator.DEFAULT_RSYNC_TIMEOUT + 900)
@alistairncoles
Copy link
Author

Tests need to be updated

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment