foo.lua

#!/usr/bin/env lua

local socket = require('socket')
local s = assert(socket.tcp())

s:connect('www.google.co.uk', 80)
s:send('GET /robots.txt HTTP/1.1\r\n\r\n')

while true do
  local line = s:receive('*l') 
  if line:match('Disallow: (.+)') then 
    local path = line:match('Disallow: (.+)')
    print('path: ' .. path)
  end
end