Fix corrupted blobs and manifests in Pulp2 ( satellite and capsule both )

check_all_blobs.sh

#!/bin/bash
mongo pulp_database --eval "db.units_docker_blob.find({},{"_id":0, "_storage_path":1}).toArray()" | grep _storage_path|cut -d '"' -f 4 > all_blob_lists_on_db.txt


for blobs in $(cat all_blob_lists_on_db.txt);
 do
 STATUS=$(sha256sum $blobs 2>&1 || echo "missing file $blobs")
 CHECKSUM_DB=$(echo $blobs|cut -d ':' -f 2)
 echo "Processing list : $blobs."
  if [[ $STATUS == *"missing file"* ]]
    then
     echo "File list missing: $blobs. Should be removed from DB"
    else
     CHECKSUM_FS=$(echo $STATUS|cut -d ' ' -f 1)
     if [ $CHECKSUM_FS != $CHECKSUM_DB ]
       then
        echo "File list corrupted: $blobs. Should be removed from DB and from the filesystem"
        echo "csdb:$CHECKSUM_DB."
        echo "csfs:$CHECKSUM_FS."
     fi
  fi
done

check_and_fix_all_blobs.sh

#!/bin/bash
mongo pulp_database --eval "db.units_docker_blob.find({},{"_id":0, "_storage_path":1}).toArray()" | grep _storage_path|cut -d '"' -f 4 > all_blob_lists_on_db.txt


for blobs in $(cat all_blob_lists_on_db.txt);
 do
 STATUS=$(sha256sum $blobs 2>&1 || echo "missing file $blobs")
 CHECKSUM_DB=$(echo $blobs|cut -d ':' -f 2)
 echo "Processing list : $blobs."
  if [[ $STATUS == *"missing file"* ]]
    then
     echo "File list missing: $blobs. Removing from DB"
     mongo pulp_database --eval "db.units_docker_blob.remove({\"digest\": \"sha256:$CHECKSUM_DB\"})"
    else
     CHECKSUM_FS=$(echo $STATUS|cut -d ' ' -f 1)
     if [ $CHECKSUM_FS != $CHECKSUM_DB ]
       then
        echo "File list corrupted: $blobs. Removing from DB and from the filesystem"
        echo "csdb:$CHECKSUM_DB."
        echo "csfs:$CHECKSUM_FS."
        rm -f $blobs
        mongo pulp_database --eval "db.units_docker_blob.remove({\"digest\": \"sha256:$CHECKSUM_DB\"})"
     fi
  fi
done

check_and_fix_manifests_and_manifest_lists.sh

#!/bin/bash
mongo pulp_database --eval "db.units_docker_manifest.find({},{"_id":0, "_storage_path":1}).toArray()"|grep _storage_path|cut -d '"' -f 4 > all_manifests_on_db.txt
mongo pulp_database --eval "db.units_docker_manifest_list.find({},{"_id":0, "_storage_path":1}).toArray()"|grep _storage_path|cut -d '"' -f 4 > all_manifest_lists_on_db.txt

for manifest in $(cat all_manifests_on_db.txt);
 do
 STATUS=$(sha256sum $manifest 2>&1 || echo "missing file $manifest")
 CHECKSUM_DB=$(echo $manifest|cut -d ':' -f 2)
 echo "Processing : $manifest."
  if [[ $STATUS == *"missing file"* ]]
    then
     echo "File missing: $manifest. Removing from DB"
     mongo pulp_database --eval "db.units_docker_manifest.remove({\"digest\": \"sha256:$CHECKSUM_DB\"})"
     mongo pulp_database --eval "db.units_docker_tag.remove({\"manifest_digest\": \"sha256:$CHECKSUM_DB\"})"
    else
     CHECKSUM_FS=$(echo $STATUS|cut -d ' ' -f 1)
     if [ $CHECKSUM_FS != $CHECKSUM_DB ]
       then
        echo "File corrupted: $manifest. Removing from DB and from the filesystem"
        echo "csdb:$CHECKSUM_DB."
        echo "csfs:$CHECKSUM_FS."
        rm -f $manifest
        mongo pulp_database --eval "db.units_docker_manifest.remove({\"digest\": \"sha256:$CHECKSUM_DB\"})"
        mongo pulp_database --eval "db.units_docker_tag.remove({\"manifest_digest\": \"sha256:$CHECKSUM_DB\"})"
     fi
  fi
done
for manifest_list in $(cat all_manifest_lists_on_db.txt);
 do
 STATUS=$(sha256sum $manifest_list 2>&1 || echo "missing file $manifest_list")
 CHECKSUM_DB=$(echo $manifest_list|cut -d ':' -f 2)
 echo "Processing list : $manifest_list."
  if [[ $STATUS == *"missing file"* ]]
    then
     echo "File list missing: $manifest_list. Removing from DB"
     mongo pulp_database --eval "db.units_docker_manifest_list.remove({\"digest\": \"sha256:$CHECKSUM_DB\"})"
     mongo pulp_database --eval "db.units_docker_tag.remove({\"manifest_digest\": \"sha256:$CHECKSUM_DB\"})"
    else
     CHECKSUM_FS=$(echo $STATUS|cut -d ' ' -f 1)
     if [ $CHECKSUM_FS != $CHECKSUM_DB ]
       then
        echo "File list corrupted: $manifest_list. Removing from DB and from the filesystem"
        echo "csdb:$CHECKSUM_DB."
        echo "csfs:$CHECKSUM_FS."
        rm -f $manifest_list
        mongo pulp_database --eval "db.units_docker_manifest_list.remove({\"digest\": \"sha256:$CHECKSUM_DB\"})"
        mongo pulp_database --eval "db.units_docker_tag.remove({\"manifest_digest\": \"sha256:$CHECKSUM_DB\"})"
     fi
  fi
done

check_manifests_and_manifest_lists.sh

#!/bin/bash
mongo pulp_database --eval "db.units_docker_manifest.find({},{"_id":0, "_storage_path":1}).toArray()"|grep _storage_path|cut -d '"' -f 4 > all_manifests_on_db.txt
mongo pulp_database --eval "db.units_docker_manifest_list.find({},{"_id":0, "_storage_path":1}).toArray()"|grep _storage_path|cut -d '"' -f 4 > all_manifest_lists_on_db.txt

for manifest in $(cat all_manifests_on_db.txt);
 do
 STATUS=$(sha256sum $manifest 2>&1 || echo "missing file $manifest")
 CHECKSUM_DB=$(echo $manifest|cut -d ':' -f 2)
 echo "Processing : $manifest."
  if [[ $STATUS == *"missing file"* ]]
    then
     echo "File missing: $manifest. Should be removed from DB"
    else
     CHECKSUM_FS=$(echo $STATUS|cut -d ' ' -f 1)
     if [ $CHECKSUM_FS != $CHECKSUM_DB ]
       then
        echo "File corrupted: $manifest. Should be removed from DB and from the filesystem"
        echo "csdb:$CHECKSUM_DB."
        echo "csfs:$CHECKSUM_FS."
     fi
  fi
done
for manifest_list in $(cat all_manifest_lists_on_db.txt);
 do
 STATUS=$(sha256sum $manifest_list 2>&1 || echo "missing file $manifest_list")
 CHECKSUM_DB=$(echo $manifest_list|cut -d ':' -f 2)
 echo "Processing list : $manifest_list."
  if [[ $STATUS == *"missing file"* ]]
    then
     echo "File list missing: $manifest_list. Should be removed from DB"
    else
     CHECKSUM_FS=$(echo $STATUS|cut -d ' ' -f 1)
     if [ $CHECKSUM_FS != $CHECKSUM_DB ]
       then
        echo "File list corrupted: $manifest_list. Should be removed from DB and from the filesystem"
        echo "csdb:$CHECKSUM_DB."
        echo "csfs:$CHECKSUM_FS."
     fi
  fi
done

• check_all_blobs.sh --> Checks docker blobs and reports any issues

• check_manifests_and_manifest_lists.sh --> Checks manifests and manifest lists and reports any issues.

• check_and_fix_all_blobs.sh --> Re-Checks corrupt\missing docker blobs and removes them from satellite\capsule so that they can be easily synced back

• check_and_fix_manifests_and_manifest_lists.sh --> Re-Checks corrupt\missing manifests and removes them from satellite\capsule so that they can be easily synced back

• Now, open a session of the main satellite and the affected capsule server and run these commands on both of the sessions at the same time, one by one.

NOTE: This can be executed without any maintenance window as well.

# chmod +x check_all_blobs.sh
# chmod +x check_manifests_and_manifest_lists.sh
# ./check_all_blobs.sh &> /tmp/check_blobs.txt
# ./check_manifests_and_manifest_lists.sh &> /tmp/check_manifests.txt

• Now, please check the /tmp/check_blobs.txt and /tmp/check_manifests.txt files.

# egrep -i "missing|corrupt" /tmp/check_blobs.txt
# egrep -i "missing|corrupt" /tmp/check_manifests.txt

• On Satellite if you get to see any issues reported, that means you have to take some actions on the satellite first before you touch the capsule.

• If no problems are reported in satellite but only on capsule the missing or corrupted files are reported, then You can only take action on Capsule.

 

** What actions to take? **

If it's Satellite then:

• On satellite,

# chmod +x check_and_fix_all_blobs.sh
# ./check_and_fix_all_blobs.sh
# chmod +x check_and_fix_manifests_and_manifest_lists.sh
# ./check_and_fix_manifests_and_manifest_lists.sh

• Simply re-sync the container\docker type repos in Satellite from the "Content --> Sync Status" page. You can sync the important ones first if you want. It's important to sync those repos properly without running into any errors or warnings.

• Re-run the check scripts and verify if any errors are reported. ( There should be none ).

# ./check_all_blobs.sh &> /tmp/new_check_blobs.txt
# ./check_manifests_and_manifest_lists.sh &> /tmp/new_check_manifests.txt
# egrep -i "missing|corrupt" /tmp/new_check_blobs.txt
# egrep -i "missing|corrupt" /tmp/new_check_manifests.txt

 

If it's NOT Satellite but only CAPSULE then :

• On capsule

# chmod +x check_and_fix_all_blobs.sh
# ./check_and_fix_all_blobs.sh
# chmod +x check_and_fix_manifests_and_manifest_lists.sh
# ./check_and_fix_manifests_and_manifest_lists.sh

• Perform an "Optimized Sync" of the affected capsule server. It's important to finish the sync successfully.

• Re-run the check scripts and verify if any errors are reported. ( There should be none ).

# ./check_all_blobs.sh &> /tmp/new_check_blobs.txt
# ./check_manifests_and_manifest_lists.sh &> /tmp/new_check_manifests.txt
# egrep -i "missing|corrupt" /tmp/new_check_blobs.txt
# egrep -i "missing|corrupt" /tmp/new_check_manifests.txt

• If all goes well, You can manually try a docker/podman pull from this capsule.