Ceph RBD Image Recovery with no running OSDs

ceph-recover-rbd-image.js

// USAGE
//
// node ceph-recover-rbd-image.js POOL_NAME IMAGE_NAME OUTPUT_IMG_FILE
// e.g
// node ceph-recover-rbd-image.js rbd kubernetes-dynamic-pvc-698abc13-ed52-4d04-a1b9-ec6674b49ac2 recoveredimage.img
// You can then mount the recovered image with:
// mkdir output
// mount recoveredimage.img output
//
// Be sure to change OSDS array below to the paths to the OSDs which contain data for the image you are
// trying to recover.
// This script assumes 4MB block size for the image.

// Add paths for each OSD you need to recover data from.
// Make sure each of these OSD services are STOPPED.
// E.g if a pool has data on both osd 0 and osd 1 but only osd 0 is corrupted, you will need to add
// both osd 0 and osd 1 here.

const OSDS = ['/var/lib/ceph/osd/ceph-0', '/var/lib/ceph/osd/ceph-1']

// The MIT License (MIT)

// Copyright (c) 2021 Jamie Greeff

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

const IMAGE_BLOCK_SIZE = 4096

// Ref:
// https://ceph-users.ceph.narkive.com/8jZ1V0Bn/recover-data-from-deleted-rbd-volume#post5
// https://hustcat.github.io/rbd-image-internal-in-ceph/
// https://chowdera.com/2021/07/20210730065234182Y.html

const { execSync } = require('child_process')
const { Uint64LE } = require('./int64-buffer.js')

const pool = process.argv[2]
const image = process.argv[3]

if (!pool || !image) throw new Error('Missing required pool and image name')

const output_img = process.argv[4]
if (!output_img) throw new Error('Missing output_img')

const exec = cmd => execSync(cmd, { maxBuffer: 1024 * 1024 * 1024 }).toString()
const execBuffer = cmd => execSync(cmd, { maxBuffer: 1024 * 1024 * 1024 })
const execJSON = cmd => JSON.parse(exec(cmd))

function listOSDObjects(dataPath) {
  return execJSON(
    `CEPH_ARGS="--bluestore-ignore-data-csum" ceph-objectstore-tool --data-path ${dataPath} --op list --format json`
  )
}

function radosObjectToOffset(objectHexString) {
  const item = parseInt(objectHexString, 16)
  const start = item * IMAGE_BLOCK_SIZE
  const end = (item + 1) * IMAGE_BLOCK_SIZE - 1
  return [start, end]
}

function cleanString(input) {
  return input.replace(/[^\d\w.\-_\:]+/g, '')
}

// fetch all objects belonging to each OSD
const allObjectByOSD = OSDS.map(dataPath => {
  console.log('Fetching OSD objects for', dataPath)
  return listOSDObjects(dataPath)
})

const allObjects = allObjectByOSD.flatMap(objects => objects)

function getObjectMetadata(oid) {
  const obj = allObjects.find(_obj => cleanString(_obj[2].oid) === oid)
  if (!obj) {
    throw new Error('Failed to find object with oid: ' + oid)
  }

  const osdDataPath = OSDS[allObjectByOSD.findIndex(objs => objs.includes(obj))]

  const pgId = obj[0]
  const objId = JSON.stringify(obj[2])

  return { dataPath: osdDataPath, pgId, objId }
}

function recoverImageMetadata(image) {
  const rbdIdObj = getObjectMetadata(`rbd_id.${image}`)

  const rbdBlockNamePart = cleanString(
    exec(
      `CEPH_ARGS="--bluestore-ignore-data-csum" ceph-objectstore-tool --data-path ${
        rbdIdObj.dataPath
      } --pgid ${rbdIdObj.pgId} ${JSON.stringify(rbdIdObj.objId)} get-bytes`
    ).trim()
  )

  const rbdHeaderObj = getObjectMetadata(`rbd_header.${rbdBlockNamePart}`)

  const imageSizeRaw = execBuffer(
    `CEPH_ARGS="--bluestore-ignore-data-csum" ceph-objectstore-tool --data-path ${
      rbdHeaderObj.dataPath
    } --pgid ${rbdHeaderObj.pgId} ${JSON.stringify(
      rbdHeaderObj.objId
    )} get-omap 'size'`
  )

  const imageSizeBytes = Uint64LE(imageSizeRaw).toNumber()

  return {
    image_size_bytes: imageSizeBytes,
    block_name_prefix: `rbd_data.${rbdBlockNamePart}`
  }
}

// recover metadata for the image
const imageMetadata = recoverImageMetadata(image)
const { block_name_prefix, image_size_bytes } = imageMetadata

if (!block_name_prefix) throw new Error('Missing block_name_prefix')

console.log('Calculated block_name_prefix as', block_name_prefix)

// get all objects for block_name_prefix by OSD
const imgObjects = allObjectByOSD.map(objects => {
  return objects.filter(obj => obj?.[2]?.oid.startsWith(block_name_prefix))
})

// merge all objects into one array and sort by object ID in ascending order
const allImgObjects = imgObjects
  .flatMap(objects => objects)
  .sort((a, b) => {
    const aOid = a[2].oid
    const bOid = b[2].oid
    if (aOid < bOid) {
      return -1
    }
    if (aOid > bOid) {
      return 1
    }
    return 0
  })

// snapid to use
// -2 seems to be the latest or equivalent to no snapshot
// then you get things like 15, 23, 29, various numbers (not sure how they are generated)
// for various snapshots. I don't think they are in any particular date order
const snapId = -2

// iterate through each object, dump data and dd write at appropriate offset
for (const obj of allImgObjects) {
  const meta = obj[2]
  if (meta.snapid !== snapId) continue

  const objectHexString = meta.oid.substring(meta.oid.lastIndexOf('.') + 1)
  const [start, end] = radosObjectToOffset(objectHexString)

  const osdDataPath = OSDS[imgObjects.findIndex(objs => objs.includes(obj))]

  const pgId = obj[0]
  const objId = JSON.stringify(meta)

  const dumpFile = `.temp-recovery-${meta.oid}.${meta.snapid}.${meta.hash}`
  exec(
    `CEPH_ARGS="--bluestore-ignore-data-csum" ceph-objectstore-tool --data-path ${osdDataPath} --pgid ${pgId} ${JSON.stringify(
      objId
    )} get-bytes > ${dumpFile}`
  )

  console.log(`Writing ${objectHexString} at ${start} - ${end}`)
  exec(`dd if=${dumpFile} of=${output_img} bs=1024 seek=${start}`)
  exec(`rm ${dumpFile}`)
}

console.log(`Resizing final .img file to ${image_size_bytes} bytes`)
exec(`dd if=/dev/zero of=${output_img} bs=1 count=0 seek=${image_size_bytes}`)

int64-buffer.js

// The MIT License (MIT)

// Copyright (c) 2015-2020 Yusuke Kawasaki

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

// int64-buffer.js

/*jshint -W018 */ // Confusing use of '!'.
/*jshint -W030 */ // Expected an assignment or function call and instead saw an expression.
/*jshint -W093 */ // Did you mean to return a conditional instead of an assignment?

var Uint64BE, Int64BE, Uint64LE, Int64LE

!(function(exports) {
  // constants

  var UNDEFINED = 'undefined'
  var BUFFER = UNDEFINED !== typeof Buffer && Buffer
  var UINT8ARRAY = UNDEFINED !== typeof Uint8Array && Uint8Array
  var ARRAYBUFFER = UNDEFINED !== typeof ArrayBuffer && ArrayBuffer
  var ZERO = [0, 0, 0, 0, 0, 0, 0, 0]
  var isArray = Array.isArray || _isArray
  var BIT32 = 4294967296
  var BIT24 = 16777216

  // storage class

  var storage // Array;

  // generate classes

  Uint64BE = factory('Uint64BE', true, true)
  Int64BE = factory('Int64BE', true, false)
  Uint64LE = factory('Uint64LE', false, true)
  Int64LE = factory('Int64LE', false, false)

  // class factory

  function factory(name, bigendian, unsigned) {
    var posH = bigendian ? 0 : 4
    var posL = bigendian ? 4 : 0
    var pos0 = bigendian ? 0 : 3
    var pos1 = bigendian ? 1 : 2
    var pos2 = bigendian ? 2 : 1
    var pos3 = bigendian ? 3 : 0
    var fromPositive = bigendian ? fromPositiveBE : fromPositiveLE
    var fromNegative = bigendian ? fromNegativeBE : fromNegativeLE
    var proto = Int64.prototype
    var isName = 'is' + name
    var _isInt64 = '_' + isName

    // properties
    proto.buffer = void 0
    proto.offset = 0
    proto[_isInt64] = true

    // methods
    proto.toNumber = toNumber
    proto.toString = toString
    proto.toJSON = toNumber
    proto.toArray = toArray

    // add .toBuffer() method only when Buffer available
    if (BUFFER) proto.toBuffer = toBuffer

    // add .toArrayBuffer() method only when Uint8Array available
    if (UINT8ARRAY) proto.toArrayBuffer = toArrayBuffer

    // isUint64BE, isInt64BE
    Int64[isName] = isInt64

    // CommonJS
    exports[name] = Int64

    return Int64

    // constructor
    function Int64(buffer, offset, value, raddix) {
      if (!(this instanceof Int64))
        return new Int64(buffer, offset, value, raddix)
      return init(this, buffer, offset, value, raddix)
    }

    // isUint64BE, isInt64BE
    function isInt64(b) {
      return !!(b && b[_isInt64])
    }

    // initializer
    function init(that, buffer, offset, value, raddix) {
      if (UINT8ARRAY && ARRAYBUFFER) {
        if (buffer instanceof ARRAYBUFFER) buffer = new UINT8ARRAY(buffer)
        if (value instanceof ARRAYBUFFER) value = new UINT8ARRAY(value)
      }

      // Int64BE() style
      if (!buffer && !offset && !value && !storage) {
        // shortcut to initialize with zero
        that.buffer = newArray(ZERO, 0)
        return
      }

      // Int64BE(value, raddix) style
      if (!isValidBuffer(buffer, offset)) {
        var _storage = storage || Array
        raddix = offset
        value = buffer
        offset = 0
        buffer = storage === BUFFER ? BUFFER.alloc(8) : new _storage(8)
      }

      that.buffer = buffer
      that.offset = offset |= 0

      // Int64BE(buffer, offset) style
      if (UNDEFINED === typeof value) return

      // Int64BE(buffer, offset, value, raddix) style
      if ('string' === typeof value) {
        fromString(buffer, offset, value, raddix || 10)
      } else if (isValidBuffer(value, raddix)) {
        fromArray(buffer, offset, value, raddix)
      } else if ('number' === typeof raddix) {
        writeInt32(buffer, offset + posH, value) // high
        writeInt32(buffer, offset + posL, raddix) // low
      } else if (value > 0) {
        fromPositive(buffer, offset, value) // positive
      } else if (value < 0) {
        fromNegative(buffer, offset, value) // negative
      } else {
        fromArray(buffer, offset, ZERO, 0) // zero, NaN and others
      }
    }

    function fromString(buffer, offset, str, raddix) {
      var pos = 0
      var len = str.length
      var high = 0
      var low = 0
      if (str[0] === '-') pos++
      var sign = pos
      while (pos < len) {
        var chr = parseInt(str[pos++], raddix)
        if (!(chr >= 0)) break // NaN
        low = low * raddix + chr
        high = high * raddix + Math.floor(low / BIT32)
        low %= BIT32
      }
      if (sign) {
        high = ~high
        if (low) {
          low = BIT32 - low
        } else {
          high++
        }
      }
      writeInt32(buffer, offset + posH, high)
      writeInt32(buffer, offset + posL, low)
    }

    function toNumber() {
      var buffer = this.buffer
      var offset = this.offset
      var high = readInt32(buffer, offset + posH)
      var low = readInt32(buffer, offset + posL)
      if (!unsigned) high |= 0 // a trick to get signed
      return high ? high * BIT32 + low : low
    }

    function toString(radix) {
      var buffer = this.buffer
      var offset = this.offset
      var high = readInt32(buffer, offset + posH)
      var low = readInt32(buffer, offset + posL)
      var str = ''
      var sign = !unsigned && high & 0x80000000
      if (sign) {
        high = ~high
        low = BIT32 - low
      }
      radix = radix || 10
      while (1) {
        var mod = (high % radix) * BIT32 + low
        high = Math.floor(high / radix)
        low = Math.floor(mod / radix)
        str = (mod % radix).toString(radix) + str
        if (!high && !low) break
      }
      if (sign) {
        str = '-' + str
      }
      return str
    }

    function writeInt32(buffer, offset, value) {
      buffer[offset + pos3] = value & 255
      value = value >> 8
      buffer[offset + pos2] = value & 255
      value = value >> 8
      buffer[offset + pos1] = value & 255
      value = value >> 8
      buffer[offset + pos0] = value & 255
    }

    function readInt32(buffer, offset) {
      return (
        buffer[offset + pos0] * BIT24 +
        (buffer[offset + pos1] << 16) +
        (buffer[offset + pos2] << 8) +
        buffer[offset + pos3]
      )
    }
  }

  function toArray(raw) {
    var buffer = this.buffer
    var offset = this.offset
    storage = null // Array

    if (raw !== false && isArray(buffer)) {
      return buffer.length === 8 ? buffer : buffer.slice(offset, offset + 8)
    }

    return newArray(buffer, offset)
  }

  function toBuffer(raw) {
    var buffer = this.buffer
    var offset = this.offset
    storage = BUFFER

    if (raw !== false && BUFFER.isBuffer(buffer)) {
      return buffer.length === 8 ? buffer : buffer.slice(offset, offset + 8)
    }

    // Buffer.from(arraybuffer) available since Node v4.5.0
    // https://nodejs.org/en/blog/release/v4.5.0/
    return BUFFER.from(toArrayBuffer.call(this, raw))
  }

  function toArrayBuffer(raw) {
    var buffer = this.buffer
    var offset = this.offset
    var arrbuf = buffer.buffer
    storage = UINT8ARRAY

    // arrbuf.slice() ignores buffer.offset until Node v8.0.0
    if (raw !== false && !buffer.offset && arrbuf instanceof ARRAYBUFFER) {
      return arrbuf.byteLength === 8 ? arrbuf : arrbuf.slice(offset, offset + 8)
    }

    var dest = new UINT8ARRAY(8)
    fromArray(dest, 0, buffer, offset)
    return dest.buffer
  }

  function isValidBuffer(buffer, offset) {
    var len = buffer && buffer.length
    offset |= 0
    return len && offset + 8 <= len && 'string' !== typeof buffer[offset]
  }

  function fromArray(destbuf, destoff, srcbuf, srcoff) {
    destoff |= 0
    srcoff |= 0
    for (var i = 0; i < 8; i++) {
      destbuf[destoff++] = srcbuf[srcoff++] & 255
    }
  }

  function newArray(buffer, offset) {
    return Array.prototype.slice.call(buffer, offset, offset + 8)
  }

  function fromPositiveBE(buffer, offset, value) {
    var pos = offset + 8
    while (pos > offset) {
      buffer[--pos] = value & 255
      value /= 256
    }
  }

  function fromNegativeBE(buffer, offset, value) {
    var pos = offset + 8
    value++
    while (pos > offset) {
      buffer[--pos] = (-value & 255) ^ 255
      value /= 256
    }
  }

  function fromPositiveLE(buffer, offset, value) {
    var end = offset + 8
    while (offset < end) {
      buffer[offset++] = value & 255
      value /= 256
    }
  }

  function fromNegativeLE(buffer, offset, value) {
    var end = offset + 8
    value++
    while (offset < end) {
      buffer[offset++] = (-value & 255) ^ 255
      value /= 256
    }
  }

  // https://github.com/retrofox/is-array
  function _isArray(val) {
    return !!val && '[object Array]' == Object.prototype.toString.call(val)
  }
})(
  typeof exports === 'object' && typeof exports.nodeName !== 'string'
    ? exports
    : this || {}
)

Use this script at your own risk.

If you create a text file of images to recover, you can do something like:

while read line
do
  echo "Recovering $line"
  node ceph-recover-rbd-image.js rbd $line $line.img
done < ./all-images-to-recover