gistfile.undefined

diff --git a/bin/mako_gc.sh b/bin/mako_gc_workaround.sh
index 3e4dfa6..33e7ffc 100755
--- a/bin/mako_gc.sh
+++ b/bin/mako_gc_workaround.sh
@@ -6,7 +6,13 @@
 #
 
 #
-# Copyright (c) 2017, Joyent, Inc.
+# Copyright (c) 2019, Joyent, Inc.
+#
+
+#
+# Note, this is a workaround script, currently only intended for manual use
+# when it is not possible to obtain a directory listing in
+# /poseidon/stor/manta_gc/mako/<storage_id>
 #
 
 ###############################################################################
@@ -20,12 +26,13 @@
 # local filesystem.  When it is done it deletes the file in manta.
 ###############################################################################
 
+
 export PATH=/opt/local/bin:$PATH
 
 
 
 ## Global vars
-
+RECORD_PATH="$1"
 # Immutables
 
 [ -z $SSH_KEY ] && SSH_KEY=/root/.ssh/id_rsa
@@ -163,8 +170,18 @@ function manta_get_to_file() {
         -H "Date: $NOW" \
         -H "Authorization: Signature $AUTHZ_HEADER,signature=\"$SIGNATURE\"" \
         -H "Connection: close" \
-        $MANTA_URL/$MANTA_USER/stor$1 >$2 \
-        || fatal "unable to get $1"
+        $MANTA_URL/$MANTA_USER/stor$1 >$2
+
+    #
+    # While failing to obtain a file is not good, we should log the error
+    # and allow the caller to decide how to proceed.
+    #
+    if [[ $? -ne 0 ]]; then
+        log "unable to get $1"
+        return 1
+    fi
+
+    return 0
 }
 
 
@@ -179,7 +196,68 @@ function manta_delete() {
         || fatal "unable to delete $1"
 }
 
+#
+# Under certain circumstances, the directory of files containing instructions
+# for object deletion may grow too large to be able to obtain a directory
+# listing through traditional means.  As a result, we are unable to process
+# any of those files because we don't know their names.  As a mitigation, we
+# can obtain the file names manually by consulting the metadata tier directly
+# in order to recreate what the results of what a successul directory listing
+# would have looked like.  This function processes those files from a local
+# source, specified by a path.  From there, we can proceed normally through the
+# rest of the garbage collection process.
+#
+function process_file() {
+    local RECORDS="$1"
+    local ret=0
+
+    while read -r line
+    do
+        FILE="$line"
+        MFILE=$MPATH/$FILE
+        LFILE=$TMP_DIR/$FILE
+
+        manta_get_to_file $MFILE $LFILE
+
+        if [[ $? -ne 0 ]]; then
+            ret=1
+            continue
+        fi
+
+        log "Processing manta object $MFILE"
+
+        while read -r LINE
+        do
+            #Filter out any lines that aren't meant for this storage node...
+            if [[ ! $LINE =~ mako.*$MANTA_STORAGE_ID ]]
+            then
+                continue
+            fi
+            log "Processing $LINE"
+            #Fields 3 and 4 are the owner and object ids, respectively.
+            ARR=($LINE)
+            OBJECT=/manta/${ARR[2]}/${ARR[3]}
+            if [[ -f $OBJECT ]]
+            then
+                auditRow "false" "$OBJECT" "$TOMB_DIR"
+                mv $OBJECT $TOMB_DIR
+                [[ $? -eq 0 ]] || fatal "Couldn't move $OBJECT"
+                ((OBJECT_COUNT++))
+            else
+                auditRow "true" "$OBJECT" "$TOMB_DIR"
+            fi
+        done < "$LFILE"
+
+        rm $LFILE
+        [[ $? -eq 0 ]] || fatal "Unable to rm $LFILE. Something is wrong."
+        manta_delete $MFILE
+
+        ((FILE_COUNT++))
 
+        log "success processing $MFILE."
+    done < "$RECORDS"
+    return $ret
+}
 
 ## Main
 
@@ -198,22 +276,31 @@ fi
 
 echo -n $PID >$PID_FILE
 
+if [[ -z "$RECORD_PATH" ]]
+then
+    fatal "No path specified."
+fi
+
+if [[ ! -d "$RECORD_PATH" ]]
+then
+    fatal "$RECORD_PATH does not exist."
+fi
+
 # Ok, we're good to start gc
 log "starting gc"
 
-GET_RES=`manta_get_no_fatal $MPATH`
-if [[ $? -ne 0 ]]
-then
-    if [[ "$GET_RES" == *404* ]]
-    then
-        log "GC not ready yet: $MPATH $GET_RES"
-        ERROR="false"
-        audit
-        exit 0
-    else
-        fatal "$MPATH $GET_RES"
+for file in "$RECORD_PATH"/*
+do
+    process_file "$file"
+
+    #
+    # Only remove the file if it appears that we successfully processed the
+    # entire thing, otherwise retain it for further analysis.
+    #
+    if [[ $? -eq 0 ]]; then
+        rm "$file"
     fi
-fi
+done
 
 # We change to nobody:nobody since that's what nginx uses for file permissions.
 # Doing that will allow us to http MOVE files via nginx to the tombstone dir.
@@ -227,51 +314,6 @@ if [ ! -d $TOMB_DIR ]; then
 fi
 mkdir -p $TMP_DIR
 
-while read -r JSON
-do
-    if [[ "$JSON" == "" ]]
-    then
-        break
-    fi
-
-    FILE=$(echo $JSON | json -a name)
-    MFILE=$MPATH/$FILE
-    LFILE=$TMP_DIR/$FILE
-    manta_get_to_file $MFILE $LFILE
-
-    log "Processing manta object $MFILE"
-
-    cat "$LFILE" | while read -r LINE
-    do
-        #Filter out any lines that aren't meant for this storage node...
-        if [[ ! $LINE =~ mako.*$MANTA_STORAGE_ID ]]
-        then
-            continue
-        fi
-        log "Processing $LINE"
-        #Fields 3 and 4 are the owner and object ids, respectively.
-        ARR=($LINE)
-        OBJECT=/manta/${ARR[2]}/${ARR[3]}
-        if [[ -f $OBJECT ]]
-        then
-            auditRow "false" "$OBJECT" "$TOMB_DIR"
-            mv $OBJECT $TOMB_DIR
-            [[ $? -eq 0 ]] || fatal "Couldn't move $OBJECT"
-            ((OBJECT_COUNT++))
-        else
-            auditRow "true" "$OBJECT" "$TOMB_DIR"
-        fi
-    done
-
-    rm $LFILE
-    [[ $? -eq 0 ]] || fatal "Unable to rm $LFILE.  Something is really wrong."
-    manta_delete $MFILE
-
-    ((FILE_COUNT++))
-
-    log "success processing $MFILE."
-done <<< "$GET_RES"
-
 log "starting tombstone directory cleanup"
 
 TOMB_CLEAN_COUNT=$(ls $TOMB_ROOT | sort | head -n -$TOMB_DIRS_TO_KEEP | wc -l)