X-Files: Conduit Experiment

conduit.ex

# X-Files S01E04 - Conduit
#
# Take a random string of binary data, and iterate through all files
#  in a directory and do a search for any matches
#
# https://blog.line72.net/?p=532

defmodule Conduit do

  def go(inp, path) do
    path
    |> list_files
    |> Task.async_stream(fn(f) ->
      case check_input(inp, f) do
        true ->
          IO.puts("found match: #{f}")
          f
        false -> nil
      end
    end, ordered: false, timeout: :infinity, max_concurrency: 128)
    |> Enum.reduce([], fn(x, acc) ->
      case x do
        {:ok, file} when not is_nil(file) -> [file | acc]
        _ -> acc
      end
    end)
  end

  def check_input(inp, f) do
    chunk_size = byte_size(inp)
    pattern = inp
    case File.open(f, [:binary, :read]) do
      {:ok, io} ->
        # Grab a chunk that is the size of our pattern
        #  AND grab the next chunk.
        # We can't look one chunk at a time, as we have to scan
        #  through both chunks completely before we can discard
        #  the first chunk.
        chunk1 = IO.binread(io, chunk_size)
        chunk2 = IO.binread(io, chunk_size)
        check_input(pattern, chunk1, chunk2, chunk_size, io)
      {:error, reason} ->
        IO.puts("Unable to open file: #{inspect reason}")
        false
    end
  end

  def check_input(_pattern, :eof, _, _, io) do
    File.close(io)
    false
  end
  def check_input(pattern, chunk1, :eof, _, _) do
    is_match?(pattern, chunk1)
  end
  def check_input(pattern, chunk1, chunk2, chunk_size, io) do
    cond do
      is_match?(pattern, chunk1) -> true
      is_match?(pattern, chunk1 <> chunk2) -> true
      true ->
        # move chunk2 to chunk1, and read another chunk
        check_input(pattern, chunk2, IO.binread(io, chunk_size), chunk_size, io)
    end
  end

  def is_match?(needle, haystack) do
    needle_bits = bit_size(needle)
    haystack_bits = bit_size(haystack)

    if haystack_bits < needle_bits do
      false
    else
      case haystack do
        <<prefix::bitstring-size(needle_bits), _::bitstring>> when prefix == needle ->
          true
        <<_::1, rest::bitstring>> ->
          is_match?(needle, rest)
      end
    end
  end
  
  def list_files(path) do
    case File.ls(path) do
      {:ok, files} ->
        files
        |> Stream.map(&Path.join(path, &1))
        |> Stream.flat_map(fn(f) ->
          cond do
            File.regular?(f) -> [f]
            File.dir?(f) -> list_files(f)
            true -> []
          end
        end)
      {:error, _e} -> []
    end
  end
  
  def random_bitstring(bit_count \\ 2162) do
    # Calculate the number of bytes needed (round up)
    byte_count = div(bit_count + 7, 8)

    # Generate random bytes (byte_count * 8 bits)
    random_bytes = :crypto.strong_rand_bytes(byte_count)

    # Pattern match to extract exactly bit_count bits
    <<result::bitstring-size(bit_count), _rest::bitstring>> = random_bytes
    result
  end
end

bit_data = Conduit.random_bitstring(2162)
#bit_data = Conduit.random_bitstring(216)
results = Conduit.go(bit_data, "/home/dillavou/")
IO.puts("Found #{Enum.count(results)} results!")
results
|> Enum.each(fn(f) -> IO.puts("  #{f}") end)
IO.puts("Found #{Enum.count(results)} results!")