Skip to content

Instantly share code, notes, and snippets.

@anatol anatol/recover_btrfs.rb Secret
Last active Nov 20, 2019

Embed
What would you like to do?
Restore files from broken btrfs filesystem
#!/usr/bin/ruby
# scp csum_recover_script.rb root@brest:recover.rb
# https://blogs.oracle.com/wim/entry/btrfs_scrub_go_fix_corruptions
require 'fileutils'
require 'find'
require 'thread'
SOURCE_DIR = '/home/anatol/tmp/old_btrfs'
DEST_DIR = '/'
VERBOSE = false
LIMIT = nil
THREADS = 40
EXTENTS_PER_THREAD = 300 # resolve extents in separate threads
def die(msg)
puts msg
exit
end
# returns [filename, size, fraginfo]
def get_fragments(file_name)
result = []
source_file = File.join(SOURCE_DIR, file_name)
puts "Getting fragments for #{file_name}"
result << file_name
ff_out = `filefrag -v "#{source_file}"`.split("\n")
if ff_out[0] != 'Filesystem type is: 9123683e'
puts "Cannot get filefrag result for this file. Skip it."
return nil
end
# extract source_size
die('Cant find file size') unless ff_out[1] =~ / is (\d+) /
source_size = $1.to_i
die("Source file size for #{file_name} does not match filefrag") if source_size != File.size(source_file)
result << source_size
extents = []
for i in 3..ff_out.length-1 do
record = ff_out[i]
elements = record.split(':')
break if elements[0].strip == source_file
start_offset = elements[2].split('..')[0].strip.to_i
length_blocks = elements[3].strip.to_i # 4096 byte blocks!
extents << [start_offset, length_blocks]
end
die('There are no extents') if extents.empty?
puts "Found #{extents.size} extents for file #{file_name}"
result << extents
return result
end
def resolve_fragment_array(filename, array)
result = []
for e in array do
start_offset = e[0]
length_blocks = e[1]
locations = `btrfs-map-logical -l #{start_offset * 4096} /dev/sda3 2>/dev/null`.split("\n")
if locations.size == 0
puts "Hmm.... Cannot find logical offset for file #{filename}, offset #{start_offset}"
return nil
end
if locations.size > 1
#puts "Woah, we've found more than one location for offset #{start_offset}: #{locations}" if VERBOSE
end
location = locations[0]
if location =~ %r{device \(null\)$} and locations.size > 1
# we are going to remove sde soon, use replica from other device
location = locations[1]
end
die('btrfs-map-logical output is incorrect') unless location =~ %r{mirror \d+ logical (\d+) physical (\d+) device (\S+)}
logical = $1.to_i
physical = $2.to_i
device = $3
die('physical offset must be page aligned') if physical%4096 != 0
result << [device, physical/4096, length_blocks]
end
return result
end
# [filename, size, [logical_offset, length]] -> [filename, size, [device, offset, length]]
def resolve_fragments(record)
filename = record[0]
fragments = record[2]
if fragments.size > EXTENTS_PER_THREAD
# assign N requests per thread
threads = fragments.each_slice(EXTENTS_PER_THREAD).map { |part|
# sub-array => thread
Thread.new do
Thread.current[:result] = resolve_fragment_array(filename, part)
end
}
threads.map(&:join)
result = []
for t in threads do
if t[:result]
result = result + t[:result]
else
puts "One of the array subparts is empty. File #{filename}"
result = nil
break
end
end
else
result = resolve_fragment_array(filename, fragments)
end
record[2] = result ? result : []
puts "Resolved #{record[2].size} fragments for #{filename}"
end
def copy_to_destination(record)
file_name = record[0]
extents = record[2]
source_file = File.join(SOURCE_DIR, file_name)
dest_file = File.join(DEST_DIR, file_name)
source_size = record[1]
if extents.empty?
puts "No extents for file #{file_name}. Skip it."
return
end
puts "Copying #{file_name} to #{dest_file} (#{extents.size} extents)"
FileUtils.mkdir_p(File.dirname(dest_file))
for e in extents do
`dd if=#{e[0]} of="#{dest_file}" oflag=append bs=4096 skip=#{e[1]} count=#{e[2]} conv=notrunc > /dev/null 2>&1`
die("dd exit code is #{$?.to_i}") if $?.to_i != 0
end
if File.size(dest_file) < source_size
puts 'recovered file too small'
return
end
File.truncate(dest_file, source_size)
puts "Recovering finished successfully for #{file_name}"
end
# mount
`mount -o ro,degraded,skip_balance /dev/sda3 #{SOURCE_DIR}`
source = ARGV[0]
die('File does not exist') unless File.exists?(source)
source = File.absolute_path(source)
files = []
Find.find(source) { |source_file|
next if File.directory?(source_file)
short_name = source_file[(SOURCE_DIR.size+1).. -1]
dest_file = File.join(DEST_DIR, short_name)
source_size = File.size(source_file)
if File.exist?(dest_file)
dest_size = File.size(dest_file)
if source_size > dest_size
puts "File #{short_name} exists and bigger at source (source=#{source_size} dest=#{dest_size}). It seems download is not finished. Redownloading it."
File.delete(dest_file)
elsif source_size < dest_size
puts "File #{short_name} exists and bigger at destination (source=#{source_size} dest=#{dest_size}). What?"
next
else
if VERBOSE
puts "File #{short_name} is the same both on src and dest"
end
next
end
end
if source_size == 0
puts "File #{short_name} is empty, skip it."
next
end
files << short_name
if LIMIT and files.size >= LIMIT
if VERBOSE
puts "Reached limit of #{LIMIT} files. Stop traversing source directory."
break
end
end
}
puts "Need to recover #{files.size} files"
records = Queue.new
for file in files do
fraginfo = get_fragments(file)
if fraginfo
records << fraginfo
end
end
`umount #{SOURCE_DIR}`
# resolve fragments is very slow, run in with multiple threads
threads = Array.new(THREADS) do |i|
Thread.new do
while record = records.pop(true) rescue nil do
resolve_fragments(record)
copy_to_destination(record)
end
end
end
threads.map(&:join)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.