Poke around at expired objects

expirer-check.py

#!/usr/bin/env python
#
# This looks at what's in the expiring objects queue, asks the object
# servers for the current real state of the objects, and then prints it all
# out.
import argparse
import sys
import itertools

import eventlet

from swift.common.storage_policy import POLICIES
from swift.common.direct_client import direct_head_object, \
    DirectClientException
from swift.common.internal_client import InternalClient, UnexpectedResponse
from swift.container.sync import ic_conf_body
from swift.common.wsgi import ConfigString


EXP_ACCOUNT = ".expiring_objects"
SWIFT_DIR = "/etc/swift"


def inspect_object(swift, ts, acc, con, obj, extra_nodes=0):
    print("looking at %s/%s/%s" % (acc, con, obj))

    # First, figure out the storage policy so we can ask the object servers
    # about this guy
    try:
        container_metadata = {
            k.lower(): v
            for k, v in swift.get_container_metadata(acc, con).items()}
    except UnexpectedResponse as err:
        print("Container %s/%s: got error %s; skipping" % (acc, con, err))
        return

    # Now go bother each node individually and see what's up
    policy = POLICIES.get_by_name(container_metadata["x-storage-policy"])
    ring = POLICIES.get_object_ring(policy.idx, SWIFT_DIR)
    partition = ring.get_part(acc, con, obj)
    node_iter = itertools.chain(
        ring.get_part_nodes(partition),
        itertools.islice(ring.get_more_nodes(partition), extra_nodes))
    for node_idx, node in enumerate(node_iter):
        req_headers = {}
        # tell the object server not to check X-Delete-At and just show me
        # what it's got
        req_headers["X-Backend-Replication"] = "true"

        # look in the right place for the data
        req_headers["X-Backend-Storage-Policy-Index"] = str(policy.idx)

        # be filterable out of the logs
        req_headers["User-Agent"] = "expirer-list.py"

        print("====================")
        print("Asking about %s/%s/%s node %d (%s:%d/%s)" % (
            acc, con, obj, node_idx, node["ip"], node["port"], node["device"]))
        try:
            resp_headers = direct_head_object(node, partition, acc, con, obj,
                                              headers=req_headers)
        except DirectClientException as err:
            print("Got status %d (%s)" % (err.http_status,
                                          err.http_headers.get(
                                              'x-backend-timestamp')))
        except eventlet.Timeout:
            print("Timeout talking to node")
        except Exception as err:
            print("Unexpected exception: %s" % (err,))
        else:
            for key, value in sorted(resp_headers.items()):
                print("%s: %s" % (key, value))

            if ts == resp_headers.get('X-Delete-At'):
                print("\nX-Delete-At matches")
            else:
                print("\nX-Delete-At does not match (queue: %s object: %r)"
                      % (ts, resp_headers.get('X-Delete-At')))


def parse_name(obj_name):
    """
    Now we've got an object name that looks like this:

    1515008817-AUTH_test/test/slow_chunked_upload.py

    Split that up.
    """
    ts, path = obj_name.split("-", 1)
    acc, con, obj = path.split("/", 2)
    return (ts, acc, con, obj)


def list_expiring_objects(swift, skip_containers=0, skip_objects=0):
    """
    List the objects in the expirer queue.

    Returns an iterable of 4-tuples (timestamp, acc, con, obj).

    It's all just strings.
    """
    for con_item in swift.iter_containers(EXP_ACCOUNT):
        con_name = con_item["name"]
        if skip_containers > 0:
            print("skipping expirer-queue container %s" % con_name)
            skip_containers -= 1
            continue

        for obj_item in swift.iter_objects(EXP_ACCOUNT, con_name):
            obj_name = obj_item["name"]
            if skip_objects > 0:
                print("skipping expirer-queue object %s" % obj_name)
                skip_objects -= 1
                continue
            print("found %s/%s/%s" % (EXP_ACCOUNT, con_name, obj_name))
            yield parse_name(obj_name)


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-n', type=int,
                        default=10,
                        help="Number of objects to inspect")
    parser.add_argument(
        '--skip-containers', type=int, default=0,
        help="Number of expirer-queue containers to skip")
    parser.add_argument(
        '--skip-objects', type=int, default=0,
        help="Number of expirer-queue entries (objects) to skip")
    parser.add_argument(
        '--extra-nodes', type=int, default=0,
        help="Number of handoffs (non-primary) nodes to inspect")
    parser.add_argument('name', nargs='?')
    args = parser.parse_args()

    swift = InternalClient(ConfigString(ic_conf_body), 'test', 1)

    if args.name:
        (ts, acc, con, obj) = parse_name(args.name)
        inspect_object(swift, ts, acc, con, obj, extra_nodes=args.extra_nodes)
        return 0

    limit = args.n
    inspected = 0

    for ts, acc, con, obj in list_expiring_objects(
            swift, skip_containers=args.skip_containers,
            skip_objects=args.skip_objects):
        inspect_object(swift, ts, acc, con, obj)
        inspected += 1
        if inspected >= limit:
            print("\nInspected %d objects; exiting as requested"
                  % (inspected,))
            return 0


if __name__ == "__main__":
    sys.exit(main())