-
-
Save bararchy/08d06b98871a1adc0abc069e6773b98d to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
abstract class IO | |
def gets_to_end : String | |
String.build do |str| | |
if decoder = decoder() | |
loop do | |
decoder.read(self) | |
break if decoder.out_slice.empty? | |
decoder.write(str) | |
end | |
else | |
buffer = uninitialized UInt8[32768] | |
while (read_bytes = read(buffer.to_slice)) > 0 | |
STDOUT.puts("read #{read_bytes} bytes") | |
str.write buffer.to_slice[0, read_bytes] | |
sleep(0.01) # Don't hog the CPU | |
end | |
end | |
end | |
end | |
def skip(bytes_count : Int) : Nil | |
buffer = uninitialized UInt8[32768] | |
while bytes_count > 0 | |
read_count = read(buffer.to_slice[0, Math.min(bytes_count, 32768)]) | |
raise IO::EOFError.new if read_count == 0 | |
bytes_count -= read_count | |
end | |
end | |
def skip_to_end : Nil | |
buffer = uninitialized UInt8[32768] | |
while read(buffer.to_slice) > 0 | |
end | |
end | |
def self.copy(src, dst) : Int64 | |
buffer = uninitialized UInt8[32768] | |
count = 0_i64 | |
while (len = src.read(buffer.to_slice).to_i32) > 0 | |
dst.write buffer.to_slice[0, len] | |
count &+= len | |
end | |
count | |
end | |
def self.copy(src, dst, limit : Int) : Int64 | |
raise ArgumentError.new("Negative limit") if limit < 0 | |
limit = limit.to_i64 | |
buffer = uninitialized UInt8[32768] | |
remaining = limit | |
while (len = src.read(buffer.to_slice[0, Math.min(buffer.size, Math.max(remaining, 0))])) > 0 | |
dst.write buffer.to_slice[0, len] | |
remaining &-= len | |
end | |
limit - remaining | |
end | |
end | |
class HTTP::Client | |
class Response | |
def self.from_io?(io, ignore_body = false, decompress = true, &block) | |
line = io.gets(32768, chomp: true) | |
return yield nil unless line | |
pieces = line.split(3) | |
raise "Invalid HTTP response" if pieces.size < 2 | |
http_version = pieces[0] | |
raise "Unsupported HTTP version: #{http_version}" unless HTTP::SUPPORTED_VERSIONS.includes?(http_version) | |
status_code = pieces[1].to_i? | |
unless status_code && 100 <= status_code < 1000 | |
raise "Invalid HTTP status code: #{pieces[1]}" | |
end | |
status = HTTP::Status.new(status_code) | |
status_message = pieces[2]? ? pieces[2].chomp : "" | |
body_type = HTTP::BodyType::OnDemand | |
body_type = HTTP::BodyType::Mandatory if mandatory_body?(status) | |
body_type = HTTP::BodyType::Prohibited if ignore_body | |
HTTP.parse_headers_and_body(io, body_type: body_type, decompress: decompress) do |headers, body| | |
return yield new status, nil, headers, status_message, http_version, body | |
end | |
nil | |
end | |
end | |
end | |
abstract class Digest | |
def update(io : IO) : self | |
buffer = uninitialized UInt8[32768] | |
while (read_bytes = io.read(buffer.to_slice)) > 0 | |
self << buffer.to_slice[0, read_bytes] | |
end | |
self | |
end | |
end | |
module IO::Buffered | |
@buffer_size = 32768 | |
end | |
class IO | |
private class Decoder | |
property out_slice : Bytes | |
@in_buffer : Pointer(UInt8) | |
def initialize(@encoding_options : EncodingOptions) | |
@iconv = Crystal::Iconv.new(encoding_options.name, "UTF-8", encoding_options.invalid) | |
@buffer = Bytes.new((GC.malloc_atomic(32768).as(UInt8*)), 32768) | |
@in_buffer = @buffer.to_unsafe | |
@in_buffer_left = LibC::SizeT.new(0) | |
@out_buffer = Bytes.new((GC.malloc_atomic(32768).as(UInt8*)), 32768) | |
@out_slice = Bytes.empty | |
@closed = false | |
end | |
def read(io) : Nil | |
loop do | |
return unless @out_slice.empty? | |
if @in_buffer_left == 0 | |
@in_buffer = @buffer.to_unsafe | |
@in_buffer_left = LibC::SizeT.new(io.read(@buffer)) | |
end | |
# If, after refilling the buffer, we couldn't read new bytes | |
# it means we reached the end | |
break if @in_buffer_left == 0 | |
# Convert bytes using iconv | |
out_buffer = @out_buffer.to_unsafe | |
out_buffer_left = LibC::SizeT.new(32768) | |
result = @iconv.convert(pointerof(@in_buffer), pointerof(@in_buffer_left), pointerof(out_buffer), pointerof(out_buffer_left)) | |
@out_slice = @out_buffer[0, 32768 - out_buffer_left] | |
# Check for errors | |
if result == Crystal::Iconv::ERROR | |
case Errno.value | |
when Errno::EILSEQ | |
# For an illegal sequence we just skip one byte and we'll continue next | |
@iconv.handle_invalid(pointerof(@in_buffer), pointerof(@in_buffer_left)) | |
when Errno::EINVAL | |
# EINVAL means "An incomplete multibyte sequence has been encountered in the input." | |
old_in_buffer_left = @in_buffer_left | |
# On invalid multibyte sequence we try to read more bytes | |
# to see if they complete the sequence | |
refill_in_buffer(io) | |
# If we couldn't read anything new, we raise or skip | |
if old_in_buffer_left == @in_buffer_left | |
@iconv.handle_invalid(pointerof(@in_buffer), pointerof(@in_buffer_left)) | |
end | |
else | |
# Not an error we can handle | |
end | |
# Continue decoding after an error | |
next | |
end | |
break | |
end | |
end | |
private def refill_in_buffer(io) | |
buffer_remaining = 32768 - @in_buffer_left - (@in_buffer - @buffer.to_unsafe) | |
if buffer_remaining < 64 | |
@buffer.copy_from(@in_buffer, @in_buffer_left) | |
@in_buffer = @buffer.to_unsafe | |
buffer_remaining = 32768 - @in_buffer_left | |
end | |
@in_buffer_left += LibC::SizeT.new(io.read(Slice.new(@in_buffer + @in_buffer_left, buffer_remaining))) | |
end | |
def read_byte(io) : UInt8? | |
read(io) | |
if out_slice.empty? | |
nil | |
else | |
byte = out_slice.to_unsafe.value | |
advance 1 | |
byte | |
end | |
end | |
def read_utf8(io, slice) : Int32 | |
count = 0 | |
until slice.empty? | |
read(io) | |
break if out_slice.empty? | |
available = Math.min(out_slice.size, slice.size) | |
out_slice[0, available].copy_to(slice.to_unsafe, available) | |
advance(available) | |
count += available | |
slice += available | |
end | |
count | |
end | |
def gets(io, delimiter : UInt8, limit : Int, chomp) : String? | |
read(io) | |
return nil if @out_slice.empty? | |
index = @out_slice.index(delimiter) | |
if index | |
# If we find it past the limit, limit the result | |
if index >= limit | |
index = limit | |
else | |
index += 1 | |
end | |
return gets_index(index, delimiter, chomp) | |
end | |
# Check if there's limit bytes in the out slice | |
if @out_slice.size >= limit | |
return gets_index(limit, delimiter, chomp) | |
end | |
# We need to read from the out_slice into a String until we find that byte, | |
# or until we consumed limit bytes | |
String.build do |str| | |
loop do | |
limit -= @out_slice.size | |
write str | |
read(io) | |
break if @out_slice.empty? | |
index = @out_slice.index(delimiter) | |
if index | |
if index >= limit | |
index = limit | |
else | |
index += 1 | |
end | |
write str, index | |
break | |
else | |
if limit < @out_slice.size | |
write(str, limit) | |
break | |
end | |
end | |
end | |
str.chomp!(delimiter) if chomp | |
end | |
end | |
private def gets_index(index, delimiter, chomp) | |
advance_increment = index | |
if chomp && index > 0 && @out_slice[index - 1] === delimiter | |
index -= 1 | |
if delimiter === '\n' && index > 0 && @out_slice[index - 1] === '\r' | |
index -= 1 | |
end | |
end | |
string = String.new(@out_slice[0, index]) | |
advance(advance_increment) | |
string | |
end | |
def write(io) : Nil | |
io.write @out_slice | |
@out_slice = Bytes.empty | |
end | |
def write(io, numbytes) : Nil | |
io.write @out_slice[0, numbytes] | |
@out_slice += numbytes | |
end | |
def advance(numbytes) : Nil | |
@out_slice += numbytes | |
end | |
def close : Nil | |
return if @closed | |
@closed = true | |
@iconv.close | |
end | |
def finalize | |
close | |
end | |
end | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment