Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
LVM Parser & Block Reader
# LVM Parser & Block Reader
#
# Copyright (C) 2016 Red Hat Inc
require 'optparse'
require 'ostruct'
require 'binary_struct'
### constants
LVM_PARTITION_TYPE = 142
SECTOR_SIZE = 512
LABEL_SCAN_SECTORS = 4
LVM_ID_LEN = 8
LVM_TYPE_LEN = 8
LVM_ID = "LABELONE"
PV_ID_LEN = 32
MDA_MAGIC_LEN = 16
FMTT_MAGIC = "\040\114\126\115\062\040\170\133\065\101\045\162\060\116\052\076"
# On disk label header.
LABEL_HEADER = BinaryStruct.new([
"A#{LVM_ID_LEN}", 'lvm_id',
'Q', 'sector_xl',
'L', 'crc_xl',
'L', 'offset_xl',
"A#{LVM_TYPE_LEN}", 'lvm_type'
])
# On disk physical volume header.
PV_HEADER = BinaryStruct.new([
"A#{PV_ID_LEN}", 'pv_uuid',
"Q", 'device_size_xl'
])
# On disk disk location structure.
DISK_LOCN = BinaryStruct.new([
"Q", 'offset',
"Q", 'size'
])
# On disk metadata area header.
MDA_HEADER = BinaryStruct.new([
"L", 'checksum_xl',
"A#{MDA_MAGIC_LEN}", 'magic',
"L", 'version',
"Q", 'start',
"Q", 'size'
])
# On disk raw location header, points to metadata.
RAW_LOCN = BinaryStruct.new([
"Q", 'offset',
"Q", 'size',
"L", 'checksum',
"L", 'filler'
])
module MD
HASH_START = '{'
HASH_END = '}'
ARRAY_START = '['
ARRAY_END = ']'
STRING_START = '"'
STRING_END = '"'
end
module Thin
SECTOR_SIZE = 512
THIN_MAGIC = 27022010
SPACE_MAP_ROOT_SIZE = 128
MAX_METADATA_BITMAPS = 255
SUPERBLOCK = BinaryStruct.new([
'L', 'csum',
'L', 'flags_',
'Q', 'block',
'A16', 'uuid',
'Q', 'magic',
'L', 'version',
'L', 'time',
'Q', 'trans_id',
'Q', 'metadata_snap',
"A#{SPACE_MAP_ROOT_SIZE}", 'data_space_map_root',
"A#{SPACE_MAP_ROOT_SIZE}", 'metadata_space_map_root',
'Q', 'data_mapping_root',
'Q', 'device_details_root',
'L', 'data_block_size', # in 512-byte sectors
'L', 'metadata_block_size', # in 512-byte sectors
'Q', 'metadata_nr_blocks',
'L', 'compat_flags',
'L', 'compat_ro_flags',
'L', 'incompat_flags'
])
SPACE_MAP = BinaryStruct.new([
'Q', 'nr_blocks',
'Q', 'nr_allocated',
'Q', 'bitmap_root',
'Q', 'ref_count_root'
])
DISK_NODE = BinaryStruct.new([
'L', 'csum',
'L', 'flags',
'Q', 'blocknr',
'L', 'nr_entries',
'L', 'max_entries',
'L', 'value_size',
'L', 'padding'
#'Q', 'keys'
])
INDEX_ENTRY = BinaryStruct.new([
'Q', 'blocknr',
'L', 'nr_free',
'L', 'none_free_before'
])
METADATA_INDEX = BinaryStruct.new([
'L', 'csum',
'L', 'padding',
'Q', 'blocknr'
])
BITMAP_HEADER = BinaryStruct.new([
'L', 'csum',
'L', 'notused',
'Q', 'blocknr'
])
DEVICE_DETAILS = BinaryStruct.new([
'Q', 'mapped_blocks',
'Q', 'transaction_id',
'L', 'creation_time',
'L', 'snapshotted_time'
])
MAPPING_DETAILS = BinaryStruct.new([
'Q', 'value'
])
end
### structures
class RawDisk
def initialize(path)
@file = File.open(path)
end
def seek(pos)
@file.seek pos
end
def read(len)
@file.read len
end
end
class LVDisk
attr_accessor :lv, :block_size, :extent_size
def initialize(lv)
self.lv = lv
self.block_size = 512
self.extent_size = lv.vg.extent_size * block_size
end
def size
lv.num_blocks
end
def read(pos, len)
return read_thin(pos, len) if lv.thin?
str = ''
end_pos = pos + len - 1
start_seg, end_seg = get_segs(pos, end_pos)
(start_seg..end_seg).each do |si|
seg = lv.segments[si]
seg_start = seg.block_start
srs = seg_start # segment read start
srl = seg.size # segment read length
if si == start_seg
srs = pos
srl = seg.size - (pos - seg_start)
end
if si == end_seg
srl = end_pos - srs + 1
end
str << read_seg(seg, srs, srl)
end
str
end
private
def read_thin(pos, len)
str = ''
device_id = lv.thin_segment.device_id
thin_pool = lv.thin_pool_volume
data_blks = thin_pool.metadata_volume.superblock.device_to_data(device_id, pos, len)
data_blks.each do |offset, len|
str << thin_pool.data_volume.disk.read(offset, len)
end
return str
end
def get_segs(pos, end_pos)
# start / end segment
ss = nil
es = nil
lv.segments.each_with_index do |seg, i|
ss = i if seg.address_range === pos
if seg.address_range === end_pos
raise "Segment sequence error" unless ss
es = i
break
end
end
raise "Segment range error" if !ss || !es
return ss, es
end
def read_seg(seg, start, len)
# TODO: support segment types other than linear (stripeCount = 1)
stripe = seg.stripes[0]
pos = stripe.start_address + start - seg.block_start
disk = stripe.physical_volume.disk
disk.seek pos #, IO::SEEK_SET
disk.read len
end
end
class VolumeGroup
attr_accessor :id, :name,
:extent_size, :seqno, :status,
:physical_volumes, :logical_volumes,
:disk_obj
def initialize(args={})
args.keys.each { |k| send("#{k}=".intern, args[k]) }
physical_volumes.values.each { |pv| pv.vg = self } if physical_volumes
logical_volumes.values.each { |lv| lv.vg = self } if logical_volumes
end
end
class PhysicalVolume
attr_accessor :id, :name,
:device, :device_size,
:pe_start, :pe_count,
:status, :vg
def initialize(args={})
args.keys.each { |k| send("#{k}=".intern, args[k]) }
end
def disk
@disk ||= $volume_groups.find { |vg|
vg.disk_obj.pv_header.pv_uuid == id
}.disk_obj.raw_disk
end
end
class LogicalVolume
attr_accessor :id, :name,
:segment_count, :segments,
:status, :vg
def initialize(args={})
args.keys.each { |k| send("#{k}=".intern, args[k]) }
segments.each { |seg| seg.lv = self } if segments
end
def disk
@disk ||= begin
LVDisk.new self
end
end
def num_blocks
@num_blocks ||= segments.inject(0) { |num, seg| num + seg.num_blocks }
end
# will raise exception if LogicalVolume is not thin metadata volume
def superblock
@superblock ||= Thin::SuperBlock.get self
end
def thin_segments
@thin_segments ||= segments.select { |segment| segment.thin? }
end
def thin_segment
@thin_segment ||= thin_segments.first
end
def thin?
!thin_segments.empty?
end
def thin_pool_volume
return nil unless thin?
return thin_segment.thin_pool_volume
end
def thin_pool_segments
@thin_pool_segments ||= segments.select { |segment| segment.thin_pool? }
end
def thin_pool_segment
thin_pool_segments.first
end
def thin_pool?
!thin_pool_segments.empty?
end
def metadata_volume
thin_pool_segment.metadata_volume
end
def data_volume
thin_pool_segment.data_volume
end
def set_volumes(lvs)
segments.each { |seg| seg.set_volumes lvs }
end
end
class LVSegment
attr_accessor :start_extent, :extent_count, :type,
:stripe_count, :stripes, :device_id, :lv
attr_accessor :thin_pool, :metadata, :pool
attr_accessor :thin_pool_volume, :metadata_volume, :data_volume
def initialize(args={})
args.keys.each { |k| send("#{k}=".intern, args[k]) }
stripes.each { |stripe| stripe.seg = self } if stripes
end
def start_address
@start_address ||= start_extent * lv.disk.extent_size
end
def end_address
@end_address ||= (start_extent + extent_count) * lv.disk.extent_size - 1
end
def address_range
@address_range ||= Range.new start_address, end_address, false
end
def block_start
@block_start ||= address_range.begin
end
def size
@size ||= end_address - start_address + 1
end
def num_blocks
@num_blocks ||= size / lv.disk.block_size
end
def thin?
type == 'thin'
end
def thin_pool?
type == 'thin-pool'
end
def set_metadata_volume(lvs)
@metadata_volume = lvs.find { |lv| lv.name == metadata }
end
def set_data_volume(lvs)
@data_volume = lvs.find { |lv| lv.name == pool }
end
def set_thin_pool_volume(lvs)
@thin_pool_volume = lvs.find { |lv| lv.name == thin_pool }
end
def set_volumes(lvs)
set_metadata_volume lvs
set_data_volume lvs
set_thin_pool_volume lvs
end
end
class LVSegStripe
attr_accessor :pvn, :ext, :seg
def initialize(pvn, ext)
self.pvn = pvn
self.ext = ext
end
def physical_volume
@physical_volume ||= begin
pv = seg.lv.vg.physical_volumes[pvn]
raise "Physical volume object (#{pvn}) not found in volume group" if pv.nil?
pv
end
end
def start_address
@start_address ||= (physical_volume.pe_start * seg.lv.disk.block_size) +
(ext * seg.lv.disk.extent_size)
end
end
### thin structures
module Thin
class BTree
FLAGS = { :internal => 1, :leaf => 2}
attr_accessor :root_address
def initialize(superblock, root_address, value_type)
@superblock = superblock
@root_address = root_address
@value_type = value_type
end
def root
@root ||= begin
@superblock.seek root_address
@superblock.read_struct DISK_NODE
end
end
def internal?
(root['flags'] & FLAGS[:internal]) != 0
end
def leaf?
(root['flags'] & FLAGS[:leaf]) != 0
end
def num_entries
@num_entries ||= root['nr_entries']
end
def max_entries
@max_entries ||= root['max_entries']
end
def key_base
root_address + DISK_NODE.size
end
def key_address(i)
key_base + i * 8
end
def value_base
key_address(max_entries)
end
def value_address(i)
value_base + @value_type.size * i
end
def keys
@keys ||= begin
@superblock.seek key_base
@superblock.read(num_entries * 8).unpack("Q#{num_entries}")
end
end
def entries
@entries ||= begin
@superblock.seek value_base
@superblock.read_structs @value_type, num_entries
end
end
def entry_for(key)
entries[keys.index(key)]
end
def to_h
@h ||=
Hash[0.upto(num_entries-1).collect do |i|
k = keys[i]
e = entries[i].kind_of?(BTree) ? entries[i].to_h : entries[i]
[k, e]
end]
end
def [](key)
return to_h[key]
end
end # class BTree
class DataMap < BTree
TIME_MASK = (1 << 24) - 1
def initialize(superblock, root_address)
super superblock, root_address, MAPPING_DETAILS
end
alias :device_blocks :keys
def entries
@dmentries ||= begin
super.collect do |entry|
value = entry['value']
internal? ? DataMap.new(@superblock, @superblock.md_block_address(value)) :
[extract_data_block(value), extract_time(value)]
end
end
end
def data_block(device_block)
device_blocks.reverse.each do |map_device_block|
if map_device_block <= device_block
entry = entry_for(map_device_block)
return entry.data_block(device_block) if entry.is_a?(DataMap)
raise RuntimeError, "LVM2Thin cannot find device block: #{device_block} (closest: #{map_device_block})" unless map_device_block == device_block
return entry.first
end
end
raise RuntimeError, "LVM2Thin could not find data block for #{device_block}"
end
private
def extract_data_block(value)
value >> 24
end
def extract_time(value)
value & TIME_MASK
end
end # class DataMap
class MappingTree < BTree
def initialize(superblock, root_address)
super superblock, root_address, MAPPING_DETAILS
end
def entries
@mtentries ||= begin
super.collect do |entry|
DataMap.new @superblock, @superblock.md_block_address(entry['value'])
end
end
end
def map_for(device_id)
entry_for(device_id)
end
end # class MappingTree
class DataSpaceMap
attr_accessor :struct
def initialize(superblock, struct)
@superblock = superblock
@struct = struct
end
def btree_root_address
@btree_root_address ||= @superblock.md_block_address(struct['bitmap_root'])
end
def btree
@btree ||= BTree.new @superblock, btree_root_address, INDEX_ENTRY
end
end
class MetadataSpaceMap
def metadata_root_address
@metadata_root_address ||= @superblock.md_block_address(struct['bitmap_root'])
end
def root
@metadata_root ||= begin
@superblock.seek metadata_root_address
@superblock.read_struct METADATA_INDEX
end
end
def indices
@metadata_indices ||= (struct['nr_blocks'].to_f / @superblock.entries_per_block).ceil
end
def index_entries
@index_entries ||=
0.upto(indices-1).collect do |i|
address = metadata_root_address + METADATA_INDEX.size + i * INDEX_ENTRY.size
@superblock.seek address
@superblock.read_struct INDEX_ENTRY
end
end
def bitmaps
@bitmaps ||= index_entries.collect do |index_entry|
@superblock.seek @superblock.md_block_address(index_entry['blocknr'])
@superblock.read_struct BITMAP_HEADER
end
end
attr_accessor :struct
def initialize(superblock, struct)
@superblock = superblock
@struct = struct
end
end # class DataSpaceMap
class SuperBlock
attr_accessor :metadata_volume
attr_accessor :struct
def self.get(metadata_volume)
@superblock ||= begin
superblock = SuperBlock.new
superblock.metadata_volume = metadata_volume
superblock.seek 0
superblock.struct = superblock.read_struct SUPERBLOCK
raise "unknown lvm2 thin metadata magic number" if superblock.struct.magic != THIN_MAGIC
superblock
end
end
### superblock properties:
def md_block_size
@md_block_size ||= struct['metadata_block_size'] * 512 # = 4096
end
def md_block_address(blk_addr)
blk_addr * md_block_size
end
def entries_per_block
@entries_per_block ||= (md_block_size - BITMAP_HEADER.size) * 4
end
def data_block_size
@data_block_size ||= struct['data_block_size'] * 512
end
def data_block_address(blk_addr)
blk_addr * data_block_size
end
### lvm thin structures:
def data_space_map
@data_space_map ||= begin
seek SUPERBLOCK.offset('data_space_map_root')
DataSpaceMap.new self, read_struct(SPACE_MAP)
end
end
def metadata_space_map
@metadata_space_map ||= begin
seek SUPERBLOCK.offset('metadata_space_map_root')
MetadataSpaceMap.new self, read_struct(SPACE_MAP)
end
end
def data_mapping_address
@data_mapping_address ||= md_block_address(struct['data_mapping_root'])
end
def data_mapping
@data_mapping ||= MappingTree.new self, data_mapping_address
end
def device_details_address
@device_details_address ||= md_block_address(struct['device_details_root'])
end
def device_details
@device_details ||= BTree.new self, device_details_address, DEVICE_DETAILS
end
### address resolution / mapping:
def device_block(device_address)
(device_address / data_block_size).to_i
end
def device_block_offset(device_address)
device_address % data_block_size
end
# return array of tuples containing data volume addresses and lengths to
# read from them to read the specified device offset & length
def device_to_data(device_id, pos, len)
dev_blk = device_block(pos)
dev_off = device_block_offset(pos)
total_len = 0
data_blks = []
num_data_blks = (len / data_block_size).to_i + 1
0.upto(num_data_blks - 1) do |i|
data_blk = data_mapping.map_for(device_id).data_block(dev_blk + i)
blk_start = data_blk * data_block_size
blk_len = 0
if i == 0
blk_start += dev_off
blk_len = data_block_size - dev_off - 1
elsif i == num_data_blks - 1
blk_len = len - total_len
else
blk_len = data_block_size
end
total_len += blk_len
data_blks << [blk_start, blk_len]
end
data_blks
end
### metadata volume disk helpers:
def seek(pos)
@seek_pos = pos
end
def read(n)
@metadata_volume.disk.read @seek_pos, n
end
def read_struct(struct)
OpenStruct.new(struct.decode(@metadata_volume.disk.read(@seek_pos, struct.size)))
end
def read_structs(struct, num)
Array.new(num) do
read_struct struct
end
end
end # class SuperBlock
end # module Thin
### processing
def disks
@disks ||= []
end
optparse = OptionParser.new do |opts|
opts.on('-h', '--help', 'Display this help screen') do
puts opts
exit
end
opts.on('-d', '--disk |path|') do |disk|
disks << disk
end
end
optparse.parse!
if disks.empty?
puts "need at least one disk"
exit 1
end
$volume_groups = []
disks.each do |disk|
$disk_obj = OpenStruct.new
$disk_obj.raw_disk = RawDisk.new disk
def disk_obj
$disk_obj
end
def seek(pos)
disk_obj.raw_disk.seek pos
end
def read(len)
disk_obj.raw_disk.read len
end
def read_struct(struct)
OpenStruct.new(struct.decode(read(struct.size)))
end
### label scanner / metadata extractor
def label_sectors
(0...LABEL_SCAN_SECTORS).collect do |s|
s * SECTOR_SIZE
end
end
def is_label?(struct)
struct.lvm_id == LVM_ID
end
def labels
disk_obj.labels ||=
label_sectors.collect do |s|
seek s
struct = read_struct LABEL_HEADER
is_label?(struct) ? struct : nil
end.compact
end
def label
disk_obj.label ||= labels.first
end
def pv_header_address
disk_obj.pv_header_address ||=
(label.sector_xl * SECTOR_SIZE) + label.offset_xl
end
def _pv_header
disk_obj._pv_header ||= begin
seek pv_header_address
read_struct PV_HEADER
end
end
def _disk_locations
_pv_header.disk_locations ||= begin
locations = []
loop do
location = read_struct DISK_LOCN
break if location.offset == 0
locations << location
end
locations
end
end
def _metadata_disk_locations
_pv_header.metadata_disk_locations ||= begin
locations = []
loop do
location = read_struct DISK_LOCN
break if location.offset == 0
locations << location
end
locations
end
end
def pv_header
disk_obj.pv_header ||= begin
ph = _pv_header
_disk_locations
_metadata_disk_locations
ph
end
end
def _metadata_locations(md)
locations = []
loop do
location = read_struct RAW_LOCN
break if location.offset == 0
location.base = md.start
locations << location
end
locations
end
def metadata_headers
disk_obj.metadata_headers ||= begin
pv_header.metadata_disk_locations.collect do |loc|
seek loc.offset
md = read_struct MDA_HEADER
raise "Unknown LVM2 Magic" if md.magic != FMTT_MAGIC
md.raw_locations = _metadata_locations(md)
md
end
end
end
def _sanitize_raw_metadata(md)
md.gsub(/#.*$/, "")
.gsub("[", "[ ")
.gsub("]", " ]")
.gsub('"', ' " ')
.delete("=,")
.gsub(/\s+/, " ")
.split(' ')
end
def raw_metadata
disk_obj.raw_metadata ||= begin
metadata_headers.collect { |hdr| hdr.raw_locations }
.flatten.collect do |location|
seek location.base + location.offset
_sanitize_raw_metadata read(location.size)
end
end
end
def num_vgs
@num_vgs ||= raw_metadata.size
end
### metadata parser
$raw_metadata = nil
$raw_metadata_i = nil
def _raw_metadata
$raw_metadata ||= Array.new(raw_metadata[$raw_metadata_i])
end
def _raw_vg_name
$raw_metadata = nil
_raw_metadata.shift
end
def _parse_metadata_hash
hash = {}
name = _raw_metadata.shift
while name && name != MD::HASH_END
hash[name] = _parse_metadata_obj
name = _raw_metadata.shift
end
hash
end
def _parse_metadata_array
array = []
val = _raw_metadata.shift
while val && val != MD::ARRAY_END
array << _parse_metadata_val(val)
val = _raw_metadata.shift
end
array
end
def _parse_metadata_val(val)
if val == MD::STRING_START
return _parse_metadata_string
else
return val
end
end
def _parse_metadata_string
str = ''
word = _raw_metadata.shift
while word && word != MD::STRING_END
str << word + " "
word = _raw_metadata.shift
end
str.chomp(" ")
end
def _parse_metadata_obj
val = _raw_metadata.shift
case val
when MD::HASH_START
_parse_metadata_hash
when MD::ARRAY_START
_parse_metadata_array
else
_parse_metadata_val(val)
end
end
def metadata
disk_obj.metadata ||= begin
md = {}
0.upto(num_vgs-1) do |i|
$raw_metadata_i = i
md[_raw_vg_name] = _parse_metadata_obj
end
md
end
end
def vg_objs
disk_obj.vg_objs ||= begin
metadata.collect do |vg_name, vgobj|
VolumeGroup.new :id => vgobj['id'],
:name => vg_name,
:extent_size => vgobj['extent_size'].to_i,
:seqno => vgobj['seqno'],
:status => vgobj["status"],
:physical_volumes => _pv_objs(vgobj["physical_volumes"]),
:logical_volumes => _lv_objs(vgobj["logical_volumes"]),
:disk_obj => disk_obj
end
end
end
def _pv_objs(pv_metadata)
pvobjs = {}
pv_metadata.each do |name, pvobj|
pvobjs[name] = _pv_obj(name, pvobj)
end
pvobjs
end
def _pv_obj(name, pvobj)
PhysicalVolume.new :id => pvobj['id'].delete('-'),
:name => name,
:device => pvobj['device'],
:device_size => pvobj['dev_size'],
:pe_start => pvobj['pe_start'].to_i,
:pe_count => pvobj['pe_count'].to_i,
:status => pvobj["status"]
end
def _lv_objs(lv_metadata)
lvobjs = {}
lv_metadata.each do |name, lvobj|
lvobjs[name] = _lv_obj(name, lvobj)
end
lvobjs
end
def _lv_obj(name, lvobj)
LogicalVolume.new :id => lvobj['id'],
:name => name,
:segment_count => lvobj['segment_count'].to_i,
:status => lvobj['status'],
:segments => _lv_segments(lvobj)
end
def _lv_segments(lvobj)
(1..lvobj['segment_count'].to_i).collect { |s| _lv_segment(lvobj["segment#{s}"]) }
end
def _lv_segment(segobj)
LVSegment.new :start_extent => segobj['start_extent'].to_i,
:extent_count => segobj['extent_count'].to_i,
:type => segobj['type'],
:stripe_count => segobj['stripe_count'].to_i,
:device_id => _lv_seg_device_id(segobj),
:thin_pool => segobj['thin_pool'],
:metadata => segobj['metadata'],
:pool => segobj['pool'],
:stripes => _lv_seg_stripes(segobj)
end
def _lv_seg_device_id(segobj)
segobj['device_id'] ? segobj['device_id'].to_i : nil
end
def _lv_seg_stripes(segobj)
segobj['stripes'] ? (segobj['stripes'].each_slice(2).collect { |pv, o| LVSegStripe.new(pv, o.to_i) }) : nil
end
###
$volume_groups += vg_objs
end
logical_volumes = $volume_groups.collect { |vg| vg.logical_volumes.values }.flatten
logical_volumes.each { |lv| lv.set_volumes logical_volumes }
#puts $volume_groups.first.logical_volumes["root"].disk.read 7929824, 512
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment