png-store.rb

#!/usr/bin/ruby

# encoding: UTF-8

require 'json'
# JSON.pretty_generate(object)

require 'date'
require 'time'

require 'securerandom'
# SecureRandom.hex    #=> "eb693ec8252cd630102fd0d0fb7c3485"
# SecureRandom.hex(4) #=> "eb693123"
# SecureRandom.uuid   #=> "2d931510-d99f-494a-8c67-87feb05e1594"

require 'fileutils'
# FileUtils.mkpath '/a/b/c'
# FileUtils.cp(src, dst)
# FileUtils.mv 'oldname', 'newname'
# FileUtils.rm(path_to_image)
# FileUtils.rm_rf('dir/to/remove')

require 'digest/sha1'
# Digest::SHA1.hexdigest 'foo'
# Digest::SHA1.file(myFile).hexdigest
# Digest::SHA256.hexdigest 'message'  
# Digest::SHA256.file(myFile).hexdigest

require 'sqlite3'

require 'zlib' # To compute CRC check
# crc = Zlib::crc32("Hello") 

# -----------------------------------------------------------------------

require "/Users/pascal/Galaxy/LucilleOS/Libraries/Ruby-Libraries/Marbles.rb"

require "/Users/pascal/Galaxy/LucilleOS/Libraries/Ruby-Libraries/AionCore.rb"

=begin

The operator is an object that has meet the following signatures

    .commitBlob(blob: BinaryData) : Hash
    .filepathToContentHash(filepath) : Hash
    .readBlobErrorIfNotFound(nhash: Hash) : BinaryData
    .datablobCheck(nhash: Hash): Boolean

class Elizabeth

    def initialize()

    end

    def commitBlob(blob)
        nhash = "SHA256-#{Digest::SHA256.hexdigest(blob)}"
        XCache::set("SHA256-#{Digest::SHA256.hexdigest(blob)}", blob)
        nhash
    end

    def filepathToContentHash(filepath)
        "SHA256-#{Digest::SHA256.file(filepath).hexdigest}"
    end

    def readBlobErrorIfNotFound(nhash)
        blob = XCache::getOrNull(nhash)
        raise "[Elizabeth error: fc1dd1aa]" if blob.nil?
        blob
    end

    def datablobCheck(nhash)
        begin
            readBlobErrorIfNotFound(nhash)
            true
        rescue
            false
        end
    end

end

AionCore::commitLocationReturnHash(operator, location)
AionCore::exportHashAtFolder(operator, nhash, targetReconstructionFolderpath)

AionFsck::structureCheckAionHash(operator, nhash)

=end

# ----------------------------------------------------------------------------

=begin

https://en.wikipedia.org/wiki/Portable_Network_Graphics

PNG-Gradient.png
89504E47 0D0A1A0A 0000000D 49484452 00000080 00000044 08020000 00C625AA 3E000000 D4494441 54785EEC D1810900 3008C030 15FF3FD9 ED0C4152 FA4172F2 4DC5D6AE D84C0700 00080000 01002000 00040080 00001000 00020040 00000800 00010020 00000400 80000010 00000200 40DDBF9D 3A586118 040228B8 98A6FFFF BF6E6C49 212CA9E0 A1BDCE43 8251E365 20A33DA2 C6166DAF D71FC674 C9B4B89D CF9A3F6B F1B6BECF 8B755BED B67C458E 3846E435 FAE7B59F F3EFAD9C 8F5D076A 525FDDCF 2CEE59DD BC1E2DFE 4A7E4100 04008000 00100000 02004000 00080000 01002000 00040080 00001000 00020040 00000800 000100A0 37497F02 9902C055 91000000 0049454E 44AE4260 82

PNG-Gradient.png
prefix: 89504E47 0D0A1A0A

length: 0000000D (13)   # PNGStoreRepresentationTransforms::stringHexToInteger("0000000D")
type  : 49484452 (IHDR) # PNGStoreRepresentationTransforms::stringHexToAscii("49484452")
data  : 00000080 00000044 08020000 00
CRC   : C625AA3E

length: 000000D4 (212)  # PNGStoreRepresentationTransforms::stringHexToInteger("000000D4")
type  : 49444154 (IDAT) # PNGStoreRepresentationTransforms::stringHexToAscii("49444154")
data  : 785EECD18109003008C0 (10)
        3015FF3FD9ED0C4152FA (20)
        4172F24DC5D6AED84C07 (30)
        00000800000100200000 (40)
        04008000001000000200 (50)
        40000008000001002000 (60)
        00040080000010000002 (70)
        0040DDBF9D3A58611804 (80)
        0228B898A6FFFFBF6E6C (90)
        49212CA9E0A1BDCE4382 (100)
        51E36520A33DA2C6166D (110)
        AFD71FC674C9B4B89DCF (120)
        9A3F6BF1B6BECF8B755B (130)
        EDB67C458E3846E435FA (140)
        E7B59FF3EFAD9C8F5D07 (150)
        6A525FDDCF2CEE59DDBC (160)
        1E2DFE4A7E4100040080 (170)
        00001000000200400000 (180)
        08000001002000000400 (190)
        80000010000002004000 (200)
        000800000100A037497F (210)
        0299 (212)
crc   : 02C05591

length: 00000000
type  : 49454E44 (IEND)
data  :
crc   : AE426082

=end

=begin

https://apidock.com/ruby/String/unpack
     # Remember: a Byte is 8bits, 2 hexadeximal values
https://ruby-doc.org/core-2.1.1/Array.html#method-i-pack

https://ruby-doc.org/core-2.5.0/IO.html#method-i-read
f: filehandler instance
f.read([length [, outbuf]]) # Reads length bytes from the I/O stream.

seek(amount, whence=IO::SEEK_SET) → 0
Seeks to a given offset anInteger in the stream according to the value of whence:

:CUR or IO::SEEK_CUR  | Seeks to _amount_ plus current position
----------------------+--------------------------------------------------
:END or IO::SEEK_END  | Seeks to _amount_ plus end of stream (you
                      | probably want a negative value for _amount_)
----------------------+--------------------------------------------------
:SET or IO::SEEK_SET  | Seeks to the absolute location given by _amount_

=end

# ----------------------------------------------------------------------------

class PNGStoreUtils
    # PNGStoreUtils::pascalPrivateChuckTypeName()
    def self.pascalPrivateChuckTypeName()
        [
            "a", # Lowercase indicates that the chunk is not critical (at opposition to "ancillary")
            "b", # Lowercase indicates that the chunk is private (at opposition to public)
            "C", # Third letter must be uppercase to conform to the PNG specification
            "d"  # Case indicates whether the chunk is safe to copy by editors that do not recognize it. 
                 # If lowercase, the chunk may be safely copied regardless of the extent of modifications to the file. 
                 # If uppercase, it may only be copied if the modifications have not touched any critical chunks.
        ].join()
        # 27**4 = 531,441 possibilities
    end
end

class PNGStoreRepresentationTransforms

    # -- data to ------------------------

    # PNGStoreRepresentationTransforms::dataToStringHex(data)
    def self.dataToStringHex(data)
        data.unpack('H*').join()
    end

    # PNGStoreRepresentationTransforms::dataToStringBinary(data)
    def self.dataToStringBinary(data)
        data.unpack('b*').join()
    end

    # PNGStoreRepresentationTransforms::dataToInteger(data)
    def self.dataToInteger(data)
        PNGStoreRepresentationTransforms::dataToStringHex(data).to_i(16)
    end

    # -- string hex to ------------------------

    # PNGStoreRepresentationTransforms::stringHexToInteger(str)
    def self.stringHexToInteger(str)
        str.to_i(16)
    end

    # PNGStoreRepresentationTransforms::stringHexToAscii(str)
    def self.stringHexToAscii(str)
        [str].pack('H*')
    end

    # PNGStoreRepresentationTransforms::stringHexToData(str)
    def self.stringHexToData(str)
        [str].pack('H*')
    end

    # -- integer to ------------------------

    # PNGStoreRepresentationTransforms::integerToStringHex(int)
    def self.integerToStringHex(int)
        int.to_s(16)
    end

    # PNGStoreRepresentationTransforms::integerToDataEnsure4Bytes(int)
    def self.integerToDataEnsure4Bytes(int)
        PNGStoreRepresentationTransforms::stringHexToData(PNGStoreRepresentationTransforms::integerToStringHex(int).rjust(8, "0"))
    end

    # PNGStoreRepresentationTransforms::typeAndDataToCRCData(type, data)
    def self.typeAndDataToCRCData(type, data)
        [Zlib::crc32(type+data).to_s(16).rjust(8, "0")].pack('H*')
    end
end

class PNGStoreChunkAlgebra

    # PNGStoreChunkAlgebra::algebraicUpgradeSx01(sx01, data)
    def self.algebraicUpgradeSx01(sx01, data)
        sx01 = sx01.clone

        prefix = sx01[0]
        chunks = sx01.drop(1)
        lastchunk = chunks.pop()

        # Removing any present abCd chunk
        chunks = chunks.select{|chunk| chunk["type"] != PNGStoreUtils::pascalPrivateChuckTypeName() }

        ourchunk = {
            "length"    => PNGStoreRepresentationTransforms::integerToDataEnsure4Bytes( PNGStoreRepresentationTransforms::dataToStringHex(data).size/2 ), # data.size wasn't working, we need the length in bytes 
            "type"      => PNGStoreUtils::pascalPrivateChuckTypeName(),
            "data"      => data,
            "crc"       => PNGStoreRepresentationTransforms::typeAndDataToCRCData(PNGStoreUtils::pascalPrivateChuckTypeName(), data)
        }

        [prefix] + chunks + [ourchunk] + [lastchunk]
    end

    # PNGStoreChunkAlgebra::extractDataFromSx01(sx01)
    def self.extractDataFromSx01(sx01)
        sx01.shift
        sx01
            .map{|chunk|
                if chunk["type"] == PNGStoreUtils::pascalPrivateChuckTypeName() then
                    chunk["data"]
                else
                    nil
                end
            }
            .compact
            .join()
    end
end

class PNGIO

    # PNGIO::pngPrefixAsData()
    def self.pngPrefixAsData()
        PNGStoreRepresentationTransforms::stringHexToData("89504E470D0A1A0A")
    end

    # PNGIO::filepathToStructureSx01(filepath)
    def self.filepathToStructureSx01(filepath)
        # Sx01 = [ prefix, chunk, ..., chunk]
        # chunk {
        #     "length" => length # Data
        #     "type"   => type   # Data
        #     "data"   => data   # Data
        #     "crc"    => crc    # Data
        # }
        array = []
        f = File.open(filepath)
        array << f.read(8)
        while !f.eof? do
            length = f.read(4)
            type   = f.read(4)
            data   = f.read(PNGStoreRepresentationTransforms::dataToInteger(length))
            crc    = f.read(4)

            if PNGStoreRepresentationTransforms::typeAndDataToCRCData(type, data) != crc then
                puts "(warning) Looks like we have a malformed chunk! (#{type})" # This forces our computation to be correct
            end

            chunk = {
                "length" => length,
                "type"   => type,
                "data"   => data,
                "crc"    => crc,
            }

            array << chunk
        end
        f.close
        array
    end

    # PNGIO::sendSx01ToDisk(filepath, sx01)
    def self.sendSx01ToDisk(filepath, sx01)
        sx01 = sx01.clone
        f = File.open(filepath, "w")
        f.write(sx01.shift)
        sx01.each{|chunk|
            f.write(chunk["length"])
            f.write(chunk["type"])
            f.write(chunk["data"])
            f.write(chunk["crc"])
        }
        f.close
    end

    # PNGIO::printSx01AsTable(sx01)
    def self.printSx01AsTable(sx01)
        sx01 = sx01.clone

        puts "-------------------------------------------------"
        puts " length    | type | data             | crc32"
        puts "-------------------------------------------------"
        sx01.shift
        sx01
            .each{|chunk|
                data2 = PNGStoreRepresentationTransforms::dataToStringHex(chunk["data"])
                data3 = data2.size > 10 ? "(...)" : "     "
                data4 = "#{data2[0, 10].rjust(10)} #{data3}"
                puts "#{PNGStoreRepresentationTransforms::dataToInteger(chunk["length"]).to_s.rjust(10)} | #{chunk["type"]} | #{data4} | #{PNGStoreRepresentationTransforms::dataToInteger(chunk["crc"])}"
            }
        puts "-------------------------------------------------"
    end

    # PNGIO::extractSQLIteDataOrMakeANewFileReturnFilepath(picturefilepath)
    def self.extractSQLIteDataOrMakeANewFileReturnFilepath(picturefilepath)
        sx01 = PNGIO::filepathToStructureSx01(picturefilepath)
        data = PNGStoreChunkAlgebra::extractDataFromSx01(sx01)
        if !data.start_with?("SQLite format 3") then
            # We create the database and return the databasefilepath
            databasefilename = "#{SecureRandom.hex}.sqlite3"
            databasefilepath = "/tmp/#{databasefilename}"
            Marbles::issueNewEmptyMarbleFile(databasefilepath)
            return databasefilepath
        else
            databasefilename = "#{SecureRandom.hex}.sqlite3"
            databasefilepath = "/tmp/#{databasefilename}"
            File.open(databasefilepath, "w"){|f| f.write(data) }
            return databasefilepath
        end
    end
end

# ----------------------------------------------------------------------------

if ARGV[0] == "chunks" and ARGV[1] then
    filepath = ARGV[1]
    sx01 = PNGIO::filepathToStructureSx01(filepath)
    PNGIO::printSx01AsTable(sx01)
    exit
end

if ARGV[0] == "set-text" and ARGV[1] and ARGV[2] then
    filepath = ARGV[1]
    line = ARGV[2]
    sx01 = PNGIO::filepathToStructureSx01(filepath)
    sx01 = PNGStoreChunkAlgebra::algebraicUpgradeSx01(sx01, line)
    PNGIO::sendSx01ToDisk(filepath, sx01)
    exit
end

if ARGV[0] == "get-text" and ARGV[1] then
    filepath = ARGV[1]
    sx01 = PNGIO::filepathToStructureSx01(filepath)
    sx01.shift
    sx01.each{|chunk|
        if chunk["type"] == PNGStoreUtils::pascalPrivateChuckTypeName() then
            puts chunk["data"]
        end
    }
    exit
end

if ARGV[0] == "set-data" and ARGV[1] then
    filepath = ARGV[1]
    data = STDIN.read()
    sx01 = PNGIO::filepathToStructureSx01(filepath)
    sx01 = PNGStoreChunkAlgebra::algebraicUpgradeSx01(sx01, data)
    PNGIO::sendSx01ToDisk(filepath, sx01)
    exit
end

if ARGV[0] == "get-data" and ARGV[1] then
    filepath = ARGV[1]
    sx01 = PNGIO::filepathToStructureSx01(filepath)
    print PNGStoreChunkAlgebra::extractDataFromSx01(sx01)
    exit
end

if ARGV[0] == "kv-store" and ARGV[1] == "set-text" and ARGV[2] and ARGV[3] and ARGV[4] then
    picturefilepath = ARGV[2]
    key = ARGV[3]
    value = ARGV[4]
    databasefilepath = PNGIO::extractSQLIteDataOrMakeANewFileReturnFilepath(picturefilepath)
    Marbles::set(databasefilepath, key, value)
    sx01 = PNGIO::filepathToStructureSx01(picturefilepath)
    sx01 = PNGStoreChunkAlgebra::algebraicUpgradeSx01(sx01, IO.read(databasefilepath))
    PNGIO::sendSx01ToDisk(picturefilepath, sx01)
    exit
end

if ARGV[0] == "kv-store" and ARGV[1] == "get-text" and ARGV[2] and ARGV[3] then
    picturefilepath = ARGV[2]
    key = ARGV[3]
    databasefilepath = PNGIO::extractSQLIteDataOrMakeANewFileReturnFilepath(picturefilepath)
    puts Marbles::get(databasefilepath, key)
    exit
end

if ARGV[0] == "ca-store" and ARGV[1] == "set-text" and ARGV[2] and ARGV[3] then
    picturefilepath = ARGV[2]
    value = ARGV[3]
    key = Digest::SHA256.hexdigest(value)
    databasefilepath = PNGIO::extractSQLIteDataOrMakeANewFileReturnFilepath(picturefilepath)
    Marbles::set(databasefilepath, key, value)
    sx01 = PNGIO::filepathToStructureSx01(picturefilepath)
    sx01 = PNGStoreChunkAlgebra::algebraicUpgradeSx01(sx01, IO.read(databasefilepath))
    PNGIO::sendSx01ToDisk(picturefilepath, sx01)
    puts key
    exit
end

if ARGV[0] == "ca-store" and ARGV[1] == "get-text" and ARGV[2] and ARGV[3] then
    picturefilepath = ARGV[2]
    key = ARGV[3]
    databasefilepath = PNGIO::extractSQLIteDataOrMakeANewFileReturnFilepath(picturefilepath)
    puts Marbles::get(databasefilepath, key)
    exit
end

if ARGV[0] == "aion" and ARGV[1] == "put-location" and ARGV[2] and ARGV[3] then
    picturefilepath = ARGV[2]
    location = ARGV[3]
    databasefilepath = PNGIO::extractSQLIteDataOrMakeANewFileReturnFilepath(picturefilepath)
    operator = MarblesElizabeth.new(databasefilepath)
    puts AionCore::commitLocationReturnHash(operator, location)
    sx01 = PNGIO::filepathToStructureSx01(picturefilepath)
    sx01 = PNGStoreChunkAlgebra::algebraicUpgradeSx01(sx01, IO.read(databasefilepath))
    PNGIO::sendSx01ToDisk(picturefilepath, sx01)
    exit
end

if ARGV[0] == "aion" and ARGV[1] == "get-blob" and ARGV[2] and ARGV[3] then
    picturefilepath = ARGV[2]
    nhash = ARGV[3]
    databasefilepath = PNGIO::extractSQLIteDataOrMakeANewFileReturnFilepath(picturefilepath)
    puts Marbles::readBlobErrorIfNotFound(databasefilepath, nhash)
    exit
end

if ARGV[0] == "aion" and ARGV[1] == "export" and ARGV[2] and ARGV[3] then
    picturefilepath = ARGV[2]
    nhash = ARGV[3]
    databasefilepath = PNGIO::extractSQLIteDataOrMakeANewFileReturnFilepath(picturefilepath)
    operator = MarblesElizabeth.new(databasefilepath)
    AionCore::exportHashAtFolder(operator, nhash, "/Users/pascal/Desktop")
    exit
end

puts "usage:"

puts "    png-store chunks <path-to-picture> # show the chunks of the file"

puts "    png-store set-text <path-to-picture> <line>"
puts "    png-store get-text <path-to-picture> # echo the line to STDOUT"

puts "    png-store set-data <path-to-picture> # read from STDIN and insert the data as added content"
puts "    png-store get-data <path-to-picture> # echo the added content to STDOUT"

puts "    png-store kv-store set-text <path-to-picture> <key> <value-line>"
puts "    png-store kv-store get-text <path-to-the-picture> <key> # echo the line to STDOUT" 

puts "    png-store ca-store set-text <path-to-picture> <value-line> # read the line and insert it in the kv-store content-addressed, return the nhash"
puts "    png-store ca-store get-text <path-to-picture> <nhash> # read the line from the kv-store against this nhash and echo to the STDOUT"

puts "    png-store aion put-location <path-to-picture> <location> # inject the location as aion structure in the kv-store"
puts "    png-store aion get-blob <path-to-picture> <nhash> # echo the blob at the  STDOUT"
puts "    png-store aion export <path-to-picture> <nhash> # export the aion hierarchy on the desktop"

puts "    png-store aion history # shows the history and interactively select a past snapshot to export"