-
-
Save Nerian/989563 to your computer and use it in GitHub Desktop.
require 'json' | |
module Resque | |
module InMemoryMerger | |
def self.merge_partial_images_from_tasks(image_hash) | |
puts 'Merging image' | |
top_partial = image_hash['0'] | |
image_hash.delete('0') | |
image_hash.each { |k,v| v.slice!(0,18) } | |
final_image = [top_partial,(image_hash.keys.sort{|a,b| a.to_i <=> b.to_i}).collect{ |i| image_hash[i] }.join].join | |
puts 'Image merged' | |
final_image | |
end | |
end | |
module Merger | |
def self.merge_partial_images_from_tasks(hash_OrderImage) | |
#puts 'Merging image' | |
partial_images_names = save_partial_images_to_tmp_folder(hash_OrderImage) | |
#puts partial_images_names.inspect | |
partial_images_names.each_cons 2 do |origin, destiny| | |
#puts "\ttail -c +19 #{origin} >> #{destiny}" | |
`tail -c +19 #{origin} >> #{destiny}` | |
end | |
#puts 'Image merged' | |
final_image = File.read(partial_images_names.last) | |
end | |
private | |
def self.save_partial_images_to_tmp_folder(partial_images) | |
files = [] | |
tmp_folder = '/tmp/dpovray/merger'+ rand(10000).to_s + '/' | |
system("mkdir -p #{tmp_folder}") | |
keys = partial_images.keys.sort{|a,b| a.to_i <=> b.to_i}.reverse | |
puts 'Keys: '+keys.inspect | |
keys.each do |key| | |
image = partial_images[key.to_s] | |
files << tmp_folder+'image'+key.to_s+'.tga' | |
File.open(tmp_folder+'image'+key.to_s+'.tga', "w") do |f| | |
f.write(image) | |
end | |
end | |
files | |
end | |
end | |
end | |
partial_image = JSON.parse(File.read(File.dirname(__FILE__)+"/code")) | |
image = Resque::Merger.merge_partial_images_from_tasks(partial_image) | |
File.open("/tmp/image-tail-merger.tga", "w") do |f| | |
f.write(image) | |
end | |
partial_image = JSON.parse(File.read(File.dirname(__FILE__)+"/code")) | |
image = Resque::InMemoryMerger.merge_partial_images_from_tasks(partial_image) | |
File.open("/tmp/image-"+Time.at(Time.now).usec.to_s+'.tga', "w") do |f| | |
f.write(image) | |
end |
Download the hash at: | |
http://dl.dropbox.com/u/834494/hash |
@diedthreetimes: I still get an invalid image.
You can see all the code in https://github.com/Nerian/DPovray The file in question is lib/resque/merger
@diedthreetimes: Oh I see the edit, I will try it now.
@diedthreetimes: Damnit, invalid image too.
@Nerian did you see my most recent change? I completely misunderstood the question to begin with. I was under the impression you where adding the 19 bytes from the current image onto the previous image.
Slice grabs what you need so you need to do slice(19,hash_OrderImage[i.to_s].size -19)
Hope this works :)
@diedthreetimes: I think I didn't explained well. I will try to be very very detailed:
I am using The Povray Rendering Engine to render an image using a cluster of machines. Each machine is going to render just a part of the total image. So I divide the total image in tasks. The hash with the tasks has a key that represents the order of the task from top ('0') to bottom('49').
The partial images are all in the TGA format. The TGA format has a magic number of 19 bytes. Each partial image is a true TGA image and so it has the 19 bytes of magic number.
Normally I would do this to get the final image:
tail -c +19 /tmp/dpovray/merger5542/image49.tga >> /tmp/dpovray/merger5542/image48.tga
tail -c +19 /tmp/dpovray/merger5542/image48.tga >> /tmp/dpovray/merger5542/image47.tga
tail -c +19 /tmp/dpovray/merger5542/image47.tga >> /tmp/dpovray/merger5542/image46.tga
tail -c +19 /tmp/dpovray/merger5542/image46.tga >> /tmp/dpovray/merger5542/image45.tga
tail -c +19 /tmp/dpovray/merger5542/image45.tga >> /tmp/dpovray/merger5542/image44.tga
tail -c +19 /tmp/dpovray/merger5542/image44.tga >> /tmp/dpovray/merger5542/image43.tga
tail -c +19 /tmp/dpovray/merger5542/image43.tga >> /tmp/dpovray/merger5542/image42.tga
tail -c +19 /tmp/dpovray/merger5542/image42.tga >> /tmp/dpovray/merger5542/image41.tga
tail -c +19 /tmp/dpovray/merger5542/image41.tga >> /tmp/dpovray/merger5542/image40.tga
tail -c +19 /tmp/dpovray/merger5542/image40.tga >> /tmp/dpovray/merger5542/image39.tga
tail -c +19 /tmp/dpovray/merger5542/image39.tga >> /tmp/dpovray/merger5542/image38.tga
tail -c +19 /tmp/dpovray/merger5542/image38.tga >> /tmp/dpovray/merger5542/image37.tga
tail -c +19 /tmp/dpovray/merger5542/image37.tga >> /tmp/dpovray/merger5542/image36.tga
tail -c +19 /tmp/dpovray/merger5542/image36.tga >> /tmp/dpovray/merger5542/image35.tga
tail -c +19 /tmp/dpovray/merger5542/image35.tga >> /tmp/dpovray/merger5542/image34.tga
tail -c +19 /tmp/dpovray/merger5542/image34.tga >> /tmp/dpovray/merger5542/image33.tga
tail -c +19 /tmp/dpovray/merger5542/image33.tga >> /tmp/dpovray/merger5542/image32.tga
tail -c +19 /tmp/dpovray/merger5542/image32.tga >> /tmp/dpovray/merger5542/image31.tga
tail -c +19 /tmp/dpovray/merger5542/image31.tga >> /tmp/dpovray/merger5542/image30.tga
tail -c +19 /tmp/dpovray/merger5542/image30.tga >> /tmp/dpovray/merger5542/image29.tga
tail -c +19 /tmp/dpovray/merger5542/image29.tga >> /tmp/dpovray/merger5542/image28.tga
tail -c +19 /tmp/dpovray/merger5542/image28.tga >> /tmp/dpovray/merger5542/image27.tga
tail -c +19 /tmp/dpovray/merger5542/image27.tga >> /tmp/dpovray/merger5542/image26.tga
tail -c +19 /tmp/dpovray/merger5542/image26.tga >> /tmp/dpovray/merger5542/image25.tga
tail -c +19 /tmp/dpovray/merger5542/image25.tga >> /tmp/dpovray/merger5542/image24.tga
tail -c +19 /tmp/dpovray/merger5542/image24.tga >> /tmp/dpovray/merger5542/image23.tga
tail -c +19 /tmp/dpovray/merger5542/image23.tga >> /tmp/dpovray/merger5542/image22.tga
tail -c +19 /tmp/dpovray/merger5542/image22.tga >> /tmp/dpovray/merger5542/image21.tga
tail -c +19 /tmp/dpovray/merger5542/image21.tga >> /tmp/dpovray/merger5542/image20.tga
tail -c +19 /tmp/dpovray/merger5542/image20.tga >> /tmp/dpovray/merger5542/image19.tga
tail -c +19 /tmp/dpovray/merger5542/image19.tga >> /tmp/dpovray/merger5542/image18.tga
tail -c +19 /tmp/dpovray/merger5542/image18.tga >> /tmp/dpovray/merger5542/image17.tga
tail -c +19 /tmp/dpovray/merger5542/image17.tga >> /tmp/dpovray/merger5542/image16.tga
tail -c +19 /tmp/dpovray/merger5542/image16.tga >> /tmp/dpovray/merger5542/image15.tga
tail -c +19 /tmp/dpovray/merger5542/image15.tga >> /tmp/dpovray/merger5542/image14.tga
tail -c +19 /tmp/dpovray/merger5542/image14.tga >> /tmp/dpovray/merger5542/image13.tga
tail -c +19 /tmp/dpovray/merger5542/image13.tga >> /tmp/dpovray/merger5542/image12.tga
tail -c +19 /tmp/dpovray/merger5542/image12.tga >> /tmp/dpovray/merger5542/image11.tga
tail -c +19 /tmp/dpovray/merger5542/image11.tga >> /tmp/dpovray/merger5542/image10.tga
tail -c +19 /tmp/dpovray/merger5542/image10.tga >> /tmp/dpovray/merger5542/image9.tga
tail -c +19 /tmp/dpovray/merger5542/image9.tga >> /tmp/dpovray/merger5542/image8.tga
tail -c +19 /tmp/dpovray/merger5542/image8.tga >> /tmp/dpovray/merger5542/image7.tga
tail -c +19 /tmp/dpovray/merger5542/image7.tga >> /tmp/dpovray/merger5542/image6.tga
tail -c +19 /tmp/dpovray/merger5542/image6.tga >> /tmp/dpovray/merger5542/image5.tga
tail -c +19 /tmp/dpovray/merger5542/image5.tga >> /tmp/dpovray/merger5542/image4.tga
tail -c +19 /tmp/dpovray/merger5542/image4.tga >> /tmp/dpovray/merger5542/image3.tga
tail -c +19 /tmp/dpovray/merger5542/image3.tga >> /tmp/dpovray/merger5542/image2.tga
tail -c +19 /tmp/dpovray/merger5542/image2.tga >> /tmp/dpovray/merger5542/image1.tga
tail -c +19 /tmp/dpovray/merger5542/image1.tga >> /tmp/dpovray/merger5542/image0.tga
...
The image5 – except the first 19 bytes – is concatenated to image4
The image4 – except the first 19 bytes – is concatenated to image3
The image3 – except the first 19 bytes – is concatenated to image2
The image2 – except the first 19 bytes – is concatenated to image1
Notice that image0 do not need to have the first 19 bytes removed.
But this is really slow when you have got a lot of files. So I want a solution that runs all in memory.
Check the gist updated code. It is all the code needed and I added more explanation at stackoverflow
@diedthreetimes: I discover how to make it right! juujuuuuuu slice!(0,18) and the array of keys must be sorted and reversed :) Woa this feelings is goood :) The refactor is awesome it makes the merge in an instant. It saves around 50 seconds for an image of 2000x2000. Thanks a lot for your help!!!
@diedthreetimes: Since my code is based on your solution and you helped me a lot I will accept your answer :) thanks!
@Nerian I'm glad it worked :) Thats odd that slice!(0,18) did the trick. According to the docs http://ruby-doc.org/core/classes/String.html#M001213 the second argument should be length and not an index. Maybe slice! opperates slightly different.
make this change, it will be faster than using pack/unpack
def self.merge_partial_images_from_tasks(hash_OrderImage)
puts 'Merging image'