Skip to content

Instantly share code, notes, and snippets.

@danneu

danneu/1-codec.clj

Created November 10, 2013 11:58
Show Gist options
  • Save danneu/7397350 to your computer and use it in GitHub Desktop.
Save danneu/7397350 to your computer and use it in GitHub Desktop.
Download ZIP
a rough implementation/dump of the bitcoin wire protocol (codec.clj) and some usage examples (chan.clj). https://github.com/ztellman/gloss/issues/27
Raw
1-codec.clj
(ns blockdude.codec
(:require [clojure.string :as str]
[gloss.core :as gloss-core :refer :all
:exclude [byte-count]]
[blockdude.hash :as hash]
[blockdude.util :refer :all]
[gloss.core.codecs :refer [identity-codec]]
[gloss.io :refer :all
:exclude [contiguous decode encode]])
(:import [java.lang Character]
[java.net InetAddress]
[java.util Random Date]))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def decode gloss.io/decode)
(def encode gloss.io/encode)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn generate-timestamp []
(long (/ (System/currentTimeMillis) 1000)))
(defn generate-time []
(-> (Date.) (.getTime) (quot 1000)))
(defn generate-nonce
"Some nonces are uint32 (32 bits),
some are uint64 (64 bits)."
[bits]
(BigInteger. bits (Random.)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn bytes->ip [bytes]
(.getHostAddress
(InetAddress/getByAddress bytes)))
(defn ip->bytes
"(ip->hex \"98.200.229.32\") -> [62 c8 e5 20]"
[ip-string]
(.getAddress (InetAddress/getByName ip-string)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Messages and payloads
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; - Var-int
;; - Message header
;; - Version payload
;; - Verack payload
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defcodec var-int-codec
(compile-frame
(header
:ubyte
;; header value -> body codec
(fn [header-byte]
;(println "h->b header-byte:" header-byte)
(case header-byte
0xfd (compile-frame :uint16-le)
0xfe (compile-frame :uint32-le)
0xff (compile-frame :uint64-le)
(compile-frame
nil-frame
;; Pre-encode
(fn [body]
;; (println "h->b pre-encode body:" body)
body)
;; Post-decode
(fn [_]
;; (println "b->h post-decode body:" _)
header-byte))))
;; Body value -> Header value
(fn [body-val]
;; (println "b->h body-val:" body-val)
(cond
(< body-val 0xfd) body-val
(<= body-val 0xffff) 0xfd
(<= body-val 0xffffffff) 0xfe
:else 0xff)))))
;; Var-str ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defcodec var-str-codec
(finite-frame var-int-codec (string :us-ascii)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn reverse-hex-bytes [hex]
(bytes->hex (into-byte-array (reverse (hex->bytes hex)))))
;; :node-network or nil
(defcodec services-codec
(enum :uint64-le
{:node-network 1}))
(def hash-codec
(compile-frame
;(string :us-ascii :length 64)
;(finite-block 32)
(finite-frame 32 (repeated :byte :prefix :none))
;; Pre-encode (Flip to little endian)
;reverse-hex-bytes
(fn [hex]
(reverse (hex->bytes hex)))
;; Post-decode (Flip to big endian
;reverse-hex-bytes
;(fn [buf])
;identity
(fn [bytez]
(bytes->hex (byte-array (reverse bytez))))))
;; Message Header (24 bytes) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; - magic :uint32-le 4
;; - command :us-ascii 12
;; - length :uint32-le 4
;; - checksum :uint32-le 4
(defcodec magic-codec (enum :uint32-le
{:main 0xd9b4bef9
:testnet 0xdab5bffa
:testnet3 0x0709110b
:namecoin 0xfeb4bef9}))
(def command-codec
(compile-frame
(string :us-ascii :length 12)
;; Pre-encode
(fn [s]
(str/join
(take 12 (concat (name s)
(repeat 12 (char 0))))))
;; Post-decode
(fn [s]
(keyword (str/join (remove (partial = (char 0)) s))))))
(defcodec length-codec :uint32-le)
(def checksum-codec
(compile-frame :uint32-be
;; Pre-encode
(fn [hex]
(BigInteger. 1 (hex->bytes hex)))
;; Post-decode
(fn [n]
(let [s (Long/toHexString n)]
(if (odd? (count s))
(str \0 s)
s)))))
(defcodec message-header-codec
(ordered-map
:magic magic-codec
:command command-codec
:length :uint32-le
:checksum checksum-codec))
;; Net-addr ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; - time :uint32-le 4
;; - services :uint64-le 8
;; - ipv6/4 :char[16] 16 (be)
;; - port :uint16-be
(def ip-codec
(compile-frame
[:byte :byte :byte :byte :byte :byte
:byte :byte :byte :byte :byte :byte
:byte :byte :byte :byte]
;; Pre-encode
(fn [ip-string]
(map unchecked-byte
(concat (repeat 10 0x00)
(repeat 2 0xff)
(ip->bytes ip-string))))
;; Post-decode
(fn [bytes]
(bytes->ip (into-byte-array (take-last 4 bytes))))))
(defcodec port-codec :uint16-be)
(defcodec net-addr-codec
(ordered-map
:services services-codec
:ip ip-codec
:port port-codec))
;; Only used in `addr` codec
(defcodec timed-net-addr-codec
(ordered-map
:time :uint32-le
:services services-codec
:ip ip-codec
:port port-codec))
;; Payloads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Version (85 bytes minimum)
;; - protocol-version :int32-le 4
;; - services :uint64-le 8
;; - timestamp :int64-le 8
;; - addr-recv :net-addr 26
;; - addr-from :net-addr 26
;; - nonce :uint64-le 8
;; - user-agent :var-str (0x00 if string is 0 bytes long)
;; - start-height :int32-le 4
;; - relay :bool 1
;; If false then broadcast transactions will not be announced
;; until a filter{load,add,clear} command is received.
;; If missing or true, no change in protocol behaviour occurs.
;;
;; TODO: Implement MISSING->TRUE (pre-encode somewhere?
;; like in version-codec?
(defcodec relay-codec
(enum :byte
{true 1
false 0}))
(defcodec user-agent-codec var-str-codec)
(defcodec version-payload-codec
(ordered-map
:protocol-version :int32-le
:services services-codec
:timestamp :uint64-le
:addr-recv net-addr-codec
:addr-from net-addr-codec
:nonce :uint64-le ; Some nonces are uint32
:user-agent var-str-codec
:start-height :int32-le
:relay relay-codec))
(defcodec verack-payload-codec
nil-frame)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(declare block-header-codec)
(defn calc-block-hash [block]
(let [block-header (merge (select-keys block [:block-version
:prev-block
:merkle-root
:timestamp
:bits
:nonce
:txn-count])
{:txn-count 0})]
(as-> (encode block-header-codec block-header) _
(contiguous _)
(.array _)
(take 80 _)
(byte-array _)
(hash/double-sha256 _)
(reverse _)
(byte-array _)
(bytes->hex _))))
;; A vector of these are embedded in a headers message.
(def block-header-codec
(compile-frame
(ordered-map
:block-version :uint32-le ; 4
:prev-block hash-codec ; 32
:merkle-root hash-codec ; 32
:timestamp :uint32-le ; 4
; 4 (aka nBits)
:bits (compile-frame
:uint32-le
;; Pre-encode
;identity
(fn [hex]
(hex->unum hex))
;; Post-decode
(fn [n]
(num->hex n)))
:nonce :uint32-le ; 4
:txn-count var-int-codec)
;; Pre-encode
identity
;; Post-decode -- Calc and append block-hash
;;identity
(fn [header]
(assoc header :block-hash (calc-block-hash header)))))
(defcodec locator-codec
(repeated hash-codec :prefix var-int-codec))
;; Responded with a headers message.
(defcodec getheaders-payload-codec
(ordered-map
:protocol-version :uint32-le
;; :hash-count var-int-codec
:locator locator-codec
:hash-stop hash-codec))
(defcodec getblocks-payload-codec
(ordered-map
:protocol-version :uint32-le
:locator locator-codec
:hash-stop hash-codec))
;; Txn ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Txn input
(defcodec txin
(ordered-map
:prev-output {:idx :uint32-le
:hash hash-codec}
:script-sig (compile-frame
(repeated :ubyte :prefix var-int-codec)
;; Pre-encode
;;identity
(fn [hex]
(as-> (hex->bytes hex) _
(map (partial bit-and 0xff) _)))
;; Post-decode
(fn [byte-seq]
(bytes->hex (byte-array
(map unchecked-byte
byte-seq)))))
:sequence :uint32-le))
;; Txn output
(defcodec txout
(ordered-map
;; Satoshis (BTC/10^8)
:value :uint64-le
:script-pubkey (compile-frame
(repeated :ubyte :prefix var-int-codec)
;; Pre-encode
;; identity
(fn [hex]
(as-> (hex->bytes hex) _
(map (partial bit-and 0xff) _)))
;; Post-decode
(fn [byte-seq]
(bytes->hex (byte-array
(map unchecked-byte
byte-seq)))))))
(declare txn)
(defn calc-txn-hash [t]
(as-> (encode txn t) _
(buf->bytes _)
(hash/double-sha256 _)
(reverse _)
(byte-array _)
(bytes->hex _)))
(defcodec txn
(compile-frame
(ordered-map
:version :uint32-le
:txins (repeated txin :prefix var-int-codec)
:txouts (repeated txout :prefix var-int-codec)
:lock-time :uint32-le)
;; Pre-encode
identity
;; Post-decode
(fn [txn-map]
(assoc txn-map :txn-hash (calc-txn-hash txn-map)))))
(defcodec block
(compile-frame
(ordered-map
:block-version :uint32-le
:prev-block hash-codec
:merkle-root hash-codec
:timestamp :uint32-le
:bits (compile-frame
:uint32-le
;; Pre-encode
identity
;; Post-decode
(fn [n]
(num->hex n)))
:nonce :uint32-le
:txns (repeated txn :prefix var-int-codec))
;; Pre-encode
identity
;; Post-decode
;;identity
(fn [block]
(assoc block :block-hash (calc-block-hash block)))
))
;; In response to getheaders.
;; It contains a vector of block-headers.
(defcodec headers-payload-codec
(repeated block-header-codec :prefix var-int-codec))
;; Inventory item (36+) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;
;; - type uint32-le 4
;; - hash char[32] 32
(defcodec inventory-codec
(ordered-map
:type (enum :uint32-le
{:error 0
:txn 1
:block 2})
:hash hash-codec))
;; 37 bytes minimum (01 - contains one hash,
;; 01 00 00 00 - type is txn
;; 32 byte hash of that txn
(defcodec inv-payload-codec
(repeated inventory-codec :prefix var-int-codec))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defcodec ping-payload-codec
{:nonce :uint64-le})
(defcodec pong-payload-codec
{:nonce :uint64-le})
(defcodec addr-payload-codec
(repeated timed-net-addr-codec :prefix var-int-codec))
(defcodec getdata-payload-codec
(repeated inventory-codec :prefix var-int-codec))
;; Full message encoding/decoding ;;;;;;;;;;;;;;;;;;;;;;;;;;
(def payload-codecs
{:addr addr-payload-codec
:getblocks getblocks-payload-codec
:getdata getdata-payload-codec
:getheaders getheaders-payload-codec
:headers headers-payload-codec
:inv inv-payload-codec
:ping ping-payload-codec
:pong pong-payload-codec
:verack verack-payload-codec
:version version-payload-codec
:block block})
(defn make-message-header [command encoded-payload]
{:magic :testnet3
:command command
:length (byte-count encoded-payload)
:checksum (bytes->hex (hash/calc-checksum encoded-payload))})
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Takes payload-map instead of encoded-payload.
(defn make-message-header2 [command payload-map]
(let [payload-codec (payload-codecs command)
encoded-payload (encode payload-codec payload-map)]
(make-message-header command encoded-payload)))
(defcodec message-codec
(compile-frame
(header message-header-codec
(fn [h]
;(println "\n3. h->pcodec h:\n" h)
(compile-frame
(finite-block (:length h))
;; Pre-encode
;identity
;; This we encode twice. Plz fix.
(fn [payload-map]
(encode (payload-codecs (:command h))
payload-map))
;; Post-decode -- Attach payload to header map
(fn [p]
(assoc h :payload p))))
;; BodyVal(payloadmap) -> Headerval
;; I.e. calc headermap from payloadmap.
;; headermap is then passed to h->bcodec to
;; get the codec to encode payloadmap.
(fn [payload-map]
;(println "-------------------------------------")
;(println "\n1. b->h payload-map:\n" payload-map)
(let [header-val
(make-message-header2
(:command (meta payload-map))
payload-map)]
;(println "\n2. b->h header-val:\n" header-val)
header-val)))
;; Pre-encode
;;
identity
;; Post-decode
identity
;; (fn [message]
;; (let [command (:command message)]
;; (update-in message
;; [:payload]
;; #(decode (payload-codecs command) %))))
))
(def ^:dynamic *host* nil)
(def ^:dynamic *port* nil)
(defmulti make-message (fn [command & _] command))
(defmethod make-message :version
[_ {:keys [host port start-height]}]
(with-meta
{:protocol-version 70001
:services :node-network
:timestamp (generate-timestamp)
:addr-recv {:services :node-network
:ip (or host *host*)
:port (or port *port*)}
:addr-from {:services :node-network
:ip "8.0.8.0"
:port 8080}
:nonce (generate-nonce 64)
:user-agent "/blockdude:0.0.1/"
:start-height start-height
:relay true}
{:command :version}))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def genesis-hash (str/join (repeat 64 \0)))
(def max-stop (str/join (repeat 64 \0)))
(defmethod make-message :getheaders
[_ {:keys [locator hash-stop]
:or {locator [genesis-hash]
hash-stop max-stop}}]
(assert (not-empty locator))
(assert (not-empty hash-stop))
(with-meta
{:protocol-version 70001
:locator locator
:hash-stop hash-stop}
{:command :getheaders}))
(defmethod make-message :getblocks
[_ {:keys [locator hash-stop]
:or {locator [genesis-hash]
hash-stop max-stop}}]
(assert (not-empty locator))
(assert (not-empty hash-stop))
(with-meta
{:protocol-version 70001
:locator locator
:hash-stop hash-stop}
{:command :getblocks}))
(defmethod make-message :ping [& _]
(with-meta
{:nonce (generate-nonce 64)}
{:command :ping}))
;; :type is :txn or :block
(defmethod make-message :getdata
[_ {:keys [type hashes]}]
(with-meta
(mapv #(assoc {:type type} :hash %) hashes)
{:command :getdata}))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; The format of a blkXXXXX.dat file is:
;; <:magic><:size><:block>
;;
;; :size includes the bytes in the :block + the 4 bytes
;; that represent the :size itself (but not :magic).
(defcodec blockdat
(ordered-map
:magic magic-codec
:size :uint32-le
:block block))
Raw
2-chan.clj
(ns blockdude.chan
(:require [lamina.core :refer :all] ;;[lamina.viz :refer :all]
[aleph.tcp :refer :all]
[blockdude.hash :as hash]
[clojure.string :as str]
[blockdude.util :refer :all]
;[blockdude.message :refer :all]
[blockdude.codec :as codec :refer :all]
[lamina.viz :refer :all]))
(def host "69.164.193.54")
(def port 18333)
;; Validation ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn valid-checksum?
"Returns true if :checksum matches the actual checksum
calculated from the payload."
[message]
(= (:checksum message)
(bytes->hex (hash/calc-checksum (:payload message)))))
(defn valid-hash-chain?
"Returns true if every :prev-block points to the previous
:block-hash given a sequence of block-header maps."
[block-headers]
(as-> block-headers _
(mapcat (juxt :prev-block :block-hash) _)
(drop 1 _)
(drop-last 1 _)
(partition 2 _)
(every? #(apply = %) _)))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn decode-payload [msg]
(let [cmd (:command msg)]
(decode (payload-codecs cmd) (:payload msg))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; 1. Connect
;; 2. Send version
;; 3. Ensure version response
;; 4. Ensure verack response
;; 5. Send getheaders
(defn handshake []
(println "-------------------------------------------")
(println "Connecting...")
(let [conn (wait-for-result
(tcp-client {:host host
:port port
:frame message-codec})
10000)]
(println "Sending version...")
(enqueue conn (make-message :version {:host host
:port port
:start-height 0}))
(println "Waiting for version...")
(let [incoming-version (wait-for-result
(read-channel conn) 5000)]
(prn incoming-version))
(println "Waiting for verack...")
(let [incoming-verack (wait-for-result
(read-channel conn) 5000)]
(prn incoming-verack))
(println "Handshake successful.")
conn))
(def conn (handshake))
conn
(enqueue conn (make-message :getheaders {}))
conn
;; Get headers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def headers-message
(wait-for-result (read-channel conn)))
(assert (valid-checksum? headers))
;; Parse the block-headers out of it
(def block-headers
(decode headers-payload-codec (:payload headers)))
(contiguous (:payload inv))
(first (decode-payload inv))
(assert (valid-hash-chain? block-headers))
;; Ping ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(enqueue conn (make-message :ping))
(def pong-response @(read-channel conn))
(decode pong-payload-codec (:payload pong-response))
;; Addr ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def addr-response @(read-channel conn))
conn
;; Get blocks ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(make-message :getblocks {})
(enqueue conn (make-message :getblocks {}))
(view-graph conn)
(def conn2 (map* valid-checksum? conn))
conn
;; Inv ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def inv @(read-channel conn))
(def header1 inv)
(def objs (decode-payload inv))
(def bhash (:hash (first objs)))
conn
(close conn)
(enqueue conn
(make-message :getdata {:type :block,
:hashes [bhash]}))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
header1
(def blk @(read-channel conn))
blk
(pprint (decode-payload blk))
;; BLOCK 1
{:txns
[{:lock-time 0,
:txouts [
{:pubkey-script 21021aeaf2f8638a129a3156fbe7e5ef635226b0bafd495ff03afe2c843d7e3a4b51ac,
:value 5000000000N}
],
:txins [
{:sequence 4294967295,
:sig-script 0420e7494d017f062f503253482f,
:prev-output {:idx 4294967295,
:hash 0000000000000000000000000000000000000000000000000000000000000000}}],
:version 1}],
:nonce 1924588547,
:bits 486604799,
:timestamp 1296688928,
:merkle-root "f0315ffc38709d70ad5647e22048358dd3745f3ce3874223c80a7c92fab0c8ba"
:prev-block "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943"
:block-version 1}
(num->hex 4294967295)
(num->hex 486604799)
"1d00ffff"
(hex->unum "1d")
;; BLOCK 2
{:block-version 1
:prev-block "00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206"
:merkle-root "20222eb90f5895556926c112bb5aa0df4ab5abc3107e21a6950aec3b2e3541e2"
:txns [{:lock-time 0,
:txouts [{:pubkey-script "21038a7f6ef1c8ca0c588aa53fa860128077c9e6c11e6830f4d7ee4e763a56b7718fac,"
:value 5000000000}],
:txins [{:sequence 4294967295,
:sig-script "0432e7494d010e062f503253482f"
:prev-output {:idx 4294967295,
:hash "0000000000000000000000000000000000000000000000000000000000000000"}}],
:version 1}],
:nonce 875942400,
:bits 486604799,
:timestamp 1296688946,
}
(Integer/toString (unchecked-byte 0x81) 2)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment