Skip to content

Instantly share code, notes, and snippets.

@bararchy

bararchy/io.cr Secret

Created September 20, 2022 10:48
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save bararchy/08d06b98871a1adc0abc069e6773b98d to your computer and use it in GitHub Desktop.
Save bararchy/08d06b98871a1adc0abc069e6773b98d to your computer and use it in GitHub Desktop.
abstract class IO
def gets_to_end : String
String.build do |str|
if decoder = decoder()
loop do
decoder.read(self)
break if decoder.out_slice.empty?
decoder.write(str)
end
else
buffer = uninitialized UInt8[32768]
while (read_bytes = read(buffer.to_slice)) > 0
STDOUT.puts("read #{read_bytes} bytes")
str.write buffer.to_slice[0, read_bytes]
sleep(0.01) # Don't hog the CPU
end
end
end
end
def skip(bytes_count : Int) : Nil
buffer = uninitialized UInt8[32768]
while bytes_count > 0
read_count = read(buffer.to_slice[0, Math.min(bytes_count, 32768)])
raise IO::EOFError.new if read_count == 0
bytes_count -= read_count
end
end
def skip_to_end : Nil
buffer = uninitialized UInt8[32768]
while read(buffer.to_slice) > 0
end
end
def self.copy(src, dst) : Int64
buffer = uninitialized UInt8[32768]
count = 0_i64
while (len = src.read(buffer.to_slice).to_i32) > 0
dst.write buffer.to_slice[0, len]
count &+= len
end
count
end
def self.copy(src, dst, limit : Int) : Int64
raise ArgumentError.new("Negative limit") if limit < 0
limit = limit.to_i64
buffer = uninitialized UInt8[32768]
remaining = limit
while (len = src.read(buffer.to_slice[0, Math.min(buffer.size, Math.max(remaining, 0))])) > 0
dst.write buffer.to_slice[0, len]
remaining &-= len
end
limit - remaining
end
end
class HTTP::Client
class Response
def self.from_io?(io, ignore_body = false, decompress = true, &block)
line = io.gets(32768, chomp: true)
return yield nil unless line
pieces = line.split(3)
raise "Invalid HTTP response" if pieces.size < 2
http_version = pieces[0]
raise "Unsupported HTTP version: #{http_version}" unless HTTP::SUPPORTED_VERSIONS.includes?(http_version)
status_code = pieces[1].to_i?
unless status_code && 100 <= status_code < 1000
raise "Invalid HTTP status code: #{pieces[1]}"
end
status = HTTP::Status.new(status_code)
status_message = pieces[2]? ? pieces[2].chomp : ""
body_type = HTTP::BodyType::OnDemand
body_type = HTTP::BodyType::Mandatory if mandatory_body?(status)
body_type = HTTP::BodyType::Prohibited if ignore_body
HTTP.parse_headers_and_body(io, body_type: body_type, decompress: decompress) do |headers, body|
return yield new status, nil, headers, status_message, http_version, body
end
nil
end
end
end
abstract class Digest
def update(io : IO) : self
buffer = uninitialized UInt8[32768]
while (read_bytes = io.read(buffer.to_slice)) > 0
self << buffer.to_slice[0, read_bytes]
end
self
end
end
module IO::Buffered
@buffer_size = 32768
end
class IO
private class Decoder
property out_slice : Bytes
@in_buffer : Pointer(UInt8)
def initialize(@encoding_options : EncodingOptions)
@iconv = Crystal::Iconv.new(encoding_options.name, "UTF-8", encoding_options.invalid)
@buffer = Bytes.new((GC.malloc_atomic(32768).as(UInt8*)), 32768)
@in_buffer = @buffer.to_unsafe
@in_buffer_left = LibC::SizeT.new(0)
@out_buffer = Bytes.new((GC.malloc_atomic(32768).as(UInt8*)), 32768)
@out_slice = Bytes.empty
@closed = false
end
def read(io) : Nil
loop do
return unless @out_slice.empty?
if @in_buffer_left == 0
@in_buffer = @buffer.to_unsafe
@in_buffer_left = LibC::SizeT.new(io.read(@buffer))
end
# If, after refilling the buffer, we couldn't read new bytes
# it means we reached the end
break if @in_buffer_left == 0
# Convert bytes using iconv
out_buffer = @out_buffer.to_unsafe
out_buffer_left = LibC::SizeT.new(32768)
result = @iconv.convert(pointerof(@in_buffer), pointerof(@in_buffer_left), pointerof(out_buffer), pointerof(out_buffer_left))
@out_slice = @out_buffer[0, 32768 - out_buffer_left]
# Check for errors
if result == Crystal::Iconv::ERROR
case Errno.value
when Errno::EILSEQ
# For an illegal sequence we just skip one byte and we'll continue next
@iconv.handle_invalid(pointerof(@in_buffer), pointerof(@in_buffer_left))
when Errno::EINVAL
# EINVAL means "An incomplete multibyte sequence has been encountered in the input."
old_in_buffer_left = @in_buffer_left
# On invalid multibyte sequence we try to read more bytes
# to see if they complete the sequence
refill_in_buffer(io)
# If we couldn't read anything new, we raise or skip
if old_in_buffer_left == @in_buffer_left
@iconv.handle_invalid(pointerof(@in_buffer), pointerof(@in_buffer_left))
end
else
# Not an error we can handle
end
# Continue decoding after an error
next
end
break
end
end
private def refill_in_buffer(io)
buffer_remaining = 32768 - @in_buffer_left - (@in_buffer - @buffer.to_unsafe)
if buffer_remaining < 64
@buffer.copy_from(@in_buffer, @in_buffer_left)
@in_buffer = @buffer.to_unsafe
buffer_remaining = 32768 - @in_buffer_left
end
@in_buffer_left += LibC::SizeT.new(io.read(Slice.new(@in_buffer + @in_buffer_left, buffer_remaining)))
end
def read_byte(io) : UInt8?
read(io)
if out_slice.empty?
nil
else
byte = out_slice.to_unsafe.value
advance 1
byte
end
end
def read_utf8(io, slice) : Int32
count = 0
until slice.empty?
read(io)
break if out_slice.empty?
available = Math.min(out_slice.size, slice.size)
out_slice[0, available].copy_to(slice.to_unsafe, available)
advance(available)
count += available
slice += available
end
count
end
def gets(io, delimiter : UInt8, limit : Int, chomp) : String?
read(io)
return nil if @out_slice.empty?
index = @out_slice.index(delimiter)
if index
# If we find it past the limit, limit the result
if index >= limit
index = limit
else
index += 1
end
return gets_index(index, delimiter, chomp)
end
# Check if there's limit bytes in the out slice
if @out_slice.size >= limit
return gets_index(limit, delimiter, chomp)
end
# We need to read from the out_slice into a String until we find that byte,
# or until we consumed limit bytes
String.build do |str|
loop do
limit -= @out_slice.size
write str
read(io)
break if @out_slice.empty?
index = @out_slice.index(delimiter)
if index
if index >= limit
index = limit
else
index += 1
end
write str, index
break
else
if limit < @out_slice.size
write(str, limit)
break
end
end
end
str.chomp!(delimiter) if chomp
end
end
private def gets_index(index, delimiter, chomp)
advance_increment = index
if chomp && index > 0 && @out_slice[index - 1] === delimiter
index -= 1
if delimiter === '\n' && index > 0 && @out_slice[index - 1] === '\r'
index -= 1
end
end
string = String.new(@out_slice[0, index])
advance(advance_increment)
string
end
def write(io) : Nil
io.write @out_slice
@out_slice = Bytes.empty
end
def write(io, numbytes) : Nil
io.write @out_slice[0, numbytes]
@out_slice += numbytes
end
def advance(numbytes) : Nil
@out_slice += numbytes
end
def close : Nil
return if @closed
@closed = true
@iconv.close
end
def finalize
close
end
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment