Created
March 5, 2019 20:37
-
-
Save jrwren/290c994978b3832cbc2027f9f2656332 to your computer and use it in GitHub Desktop.
statsassembly from https://github.com/google/gopacket/tree/master/examples/statsassembly to assembly only TLS sessions
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// Copyright 2012 Google, Inc. All rights reserved. | |
// | |
// Use of this source code is governed by a BSD-style license | |
// that can be found in the LICENSE file in the root of the source | |
// tree. | |
// This binary provides sample code for using the gopacket TCP assembler raw, | |
// without the help of the tcpreader library. It watches TCP streams and | |
// reports statistics on completed streams. | |
// | |
// It also uses gopacket.DecodingLayerParser instead of the normal | |
// gopacket.PacketSource, to highlight the methods, pros, and cons of this | |
// approach. | |
package main | |
import ( | |
"encoding/hex" | |
"flag" | |
"fmt" | |
"io" | |
"log" | |
"time" | |
"github.com/google/gopacket" | |
"github.com/google/gopacket/examples/util" | |
"github.com/google/gopacket/layers" | |
"github.com/google/gopacket/pcap" | |
"github.com/google/gopacket/tcpassembly" | |
) | |
var input = flag.String("r", "file.pcap", "pcapfile") | |
var snaplen = flag.Int("s", 65536, "SnapLen for pcap packet capture") | |
var filter = flag.String("f", "tcp", "BPF filter for pcap") | |
var logAllPackets = flag.Bool("v", false, "Log whenever we see a packet") | |
var logTLSDetails = flag.Bool("vv", false, "Log TLS details") | |
var bufferedPerConnection = flag.Int("connection_max_buffer", 0, ` | |
Max packets to buffer for a single connection before skipping over a gap in data | |
and continuing to stream the connection after the buffer. If zero or less, this | |
is infinite.`) | |
var bufferedTotal = flag.Int("total_max_buffer", 0, ` | |
Max packets to buffer total before skipping over gaps in connections and | |
continuing to stream connection data. If zero or less, this is infinite`) | |
var flushAfter = flag.String("flush_after", "2m", ` | |
Connections which have buffered packets (they've gotten packets out of order and | |
are waiting for old packets to fill the gaps) are flushed after they're this old | |
(their oldest gap is skipped). Any string parsed by time.ParseDuration is | |
acceptable here`) | |
var packetCount = flag.Int("c", -1, ` | |
Quit after processing this many packets, flushing all currently buffered | |
connections. If negative, this is infinite`) | |
// simpleStreamFactory implements tcpassembly.StreamFactory | |
type statsStreamFactory struct{} | |
// TLStatsStream will handle the actual decoding of stats requests. | |
type TLSStatsStream struct { | |
net, transport gopacket.Flow | |
bytes, packets, outOfOrder, skipped int64 | |
start, end time.Time | |
sawStart, sawEnd bool | |
data []byte | |
} | |
// New creates a new stream. It's called whenever the assembler sees a stream | |
// it isn't currently following. | |
func (factory *statsStreamFactory) New(net, transport gopacket.Flow) tcpassembly.Stream { | |
log.Printf("new stream %v:%v started", net, transport) | |
s := &TLSStatsStream{ | |
net: net, | |
transport: transport, | |
start: time.Now(), | |
} | |
s.end = s.start | |
// ReaderStream implements tcpassembly.Stream, so we can return a pointer to it. | |
return s | |
} | |
// Reassembled is called whenever new packet data is available for reading. | |
// Reassembly objects contain stream data IN ORDER. | |
func (s *TLSStatsStream) Reassembled(reassemblies []tcpassembly.Reassembly) { | |
for _, reassembly := range reassemblies { | |
if reassembly.Seen.Before(s.end) { | |
s.outOfOrder++ | |
} else { | |
s.end = reassembly.Seen | |
} | |
s.bytes += int64(len(reassembly.Bytes)) | |
s.data = append(s.data, reassembly.Bytes...) | |
s.packets += 1 | |
if reassembly.Skip > 0 { | |
s.skipped += int64(reassembly.Skip) | |
} | |
s.sawStart = s.sawStart || reassembly.Start | |
s.sawEnd = s.sawEnd || reassembly.End | |
} | |
} | |
// ReassemblyComplete is called when the TCP assembler believes a stream has | |
// finished. | |
func (s *TLSStatsStream) ReassemblyComplete() { | |
diffSecs := float64(s.end.Sub(s.start)) / float64(time.Second) | |
log.Printf("Reassembly of stream %v:%v complete - start:%v end:%v bytes:%v packets:%v ooo:%v bps:%v pps:%v skipped:%v", | |
s.net, s.transport, s.start, s.end, s.bytes, s.packets, s.outOfOrder, | |
float64(s.bytes)/diffSecs, float64(s.packets)/diffSecs, s.skipped) | |
streams = append(streams, s) | |
} | |
var streams []*TLSStatsStream | |
func main() { | |
defer util.Run()() | |
flushDuration, err := time.ParseDuration(*flushAfter) | |
if err != nil { | |
log.Fatal("invalid flush duration: ", *flushAfter) | |
} | |
// Set up pcap packet capture | |
handle, err := pcap.OpenOffline(*input) | |
if err != nil { | |
log.Fatal("error opening pcap handle: ", err) | |
} | |
if err := handle.SetBPFFilter(*filter); err != nil { | |
log.Fatal("error setting BPF filter: ", err) | |
} | |
// Set up assembly | |
streamFactory := &statsStreamFactory{} | |
streamPool := tcpassembly.NewStreamPool(streamFactory) | |
assembler := tcpassembly.NewAssembler(streamPool) | |
assembler.MaxBufferedPagesPerConnection = *bufferedPerConnection | |
assembler.MaxBufferedPagesTotal = *bufferedTotal | |
log.Println("reading in packets") | |
// We use a DecodingLayerParser here instead of a simpler PacketSource. | |
// This approach should be measurably faster, but is also more rigid. | |
// PacketSource will handle any known type of packet safely and easily, | |
// but DecodingLayerParser will only handle those packet types we | |
// specifically pass in. This trade-off can be quite useful, though, in | |
// high-throughput situations. | |
var eth layers.Ethernet | |
var dot1q layers.Dot1Q | |
var ip4 layers.IPv4 | |
var ip6 layers.IPv6 | |
var ip6extensions layers.IPv6ExtensionSkipper | |
var tcp layers.TCP | |
var payload gopacket.Payload | |
parser := gopacket.NewDecodingLayerParser(layers.LayerTypeEthernet, | |
ð, &dot1q, &ip4, &ip6, &ip6extensions, &tcp, &payload) | |
decoded := make([]gopacket.LayerType, 0, 5) | |
nextFlush := time.Now().Add(flushDuration / 2) | |
var byteCount int64 | |
start := time.Now() | |
tlsFlows := make(map[gopacket.Flow]struct{}) | |
loop: | |
for no := 0; *packetCount != 0; *packetCount-- { | |
no++ | |
// Check to see if we should flush the streams we have | |
// that haven't seen any new data in a while. Note we set a | |
// timeout on our PCAP handle, so this should happen even if we | |
// never see packet data. | |
if time.Now().After(nextFlush) { | |
stats, _ := handle.Stats() | |
log.Printf("flushing all streams that haven't seen packets in the last 2 minutes, pcap stats: %+v", stats) | |
assembler.FlushOlderThan(time.Now().Add(flushDuration)) | |
nextFlush = time.Now().Add(flushDuration / 2) | |
} | |
// To speed things up, we're also using the ZeroCopy method for | |
// reading packet data. This method is faster than the normal | |
// ReadPacketData, but the returned bytes in 'data' are | |
// invalidated by any subsequent ZeroCopyReadPacketData call. | |
// Note that tcpassembly is entirely compatible with this packet | |
// reading method. This is another trade-off which might be | |
// appropriate for high-throughput sniffing: it avoids a packet | |
// copy, but its cost is much more careful handling of the | |
// resulting byte slice. | |
data, ci, err := handle.ZeroCopyReadPacketData() | |
if err != nil { | |
if err == io.EOF { | |
break | |
} | |
log.Printf("error getting packet: %v", err) | |
continue | |
} | |
err = parser.DecodeLayers(data, &decoded) | |
if err != nil { | |
log.Printf("error decoding packet: %v", err) | |
continue | |
} | |
if *logAllPackets { | |
log.Printf("decoded the following layers: %v", decoded) | |
} | |
byteCount += int64(len(data)) | |
// Find either the IPv4 or IPv6 address to use as our network | |
// layer. | |
foundNetLayer := false | |
var netFlow gopacket.Flow | |
for _, typ := range decoded { | |
switch typ { | |
case layers.LayerTypeIPv4: | |
netFlow = ip4.NetworkFlow() | |
foundNetLayer = true | |
case layers.LayerTypeIPv6: | |
netFlow = ip6.NetworkFlow() | |
foundNetLayer = true | |
case layers.LayerTypeTCP: | |
if !foundNetLayer { | |
log.Println("could not find IPv4 or IPv6 layer, inoring") | |
continue loop | |
} | |
// Ignore empty segments. | |
if len(tcp.Payload) == 0 { | |
continue loop | |
} | |
if _, ok := tlsFlows[netFlow]; !ok { | |
var tls layers.TLS | |
err = tls.DecodeFromBytes(tcp.Payload, parser) | |
// Segments are going to be wrong length. | |
if err != nil && err.Error() != "TLS packet length mismatch" { | |
if *logTLSDetails { | |
log.Printf("wtf# %d %s %x\n", no, err, tcp.Payload[0:5]) | |
} | |
continue loop | |
} | |
if len(tls.Handshake) > 0 { | |
if *logTLSDetails { | |
log.Println("TLS Handshake", tls.Handshake) | |
} | |
continue loop | |
} | |
if len(tls.ChangeCipherSpec) > 0 { | |
if *logTLSDetails { | |
log.Println("TLS ChangeCipherSpec", tls.ChangeCipherSpec) | |
} | |
continue loop | |
} | |
tlsFlows[netFlow] = struct{}{} | |
if tcp.Payload[0] == 0x17 { // TLS Application Data | |
tcp.Payload = tcp.Payload[5:] | |
} | |
} | |
assembler.AssembleWithTimestamp(netFlow, &tcp, ci.Timestamp) | |
continue loop | |
} | |
} | |
log.Println("could not find TCP layer") | |
} | |
assembler.FlushAll() | |
log.Printf("processed %d bytes in %v", byteCount, time.Since(start)) | |
for _, stream := range streams { | |
fmt.Println(hex.Dump(stream.data)) | |
} | |
} |
Author
jrwren
commented
Jun 12, 2019
via email
Yes, if you have the TLS session key.
… On Jun 11, 2019, at 11:09 PM, Saint Tukio ***@***.***> wrote:
I found this very useful @jrwren <https://github.com/jrwren>.
Quick question, do you know how I can decrypt the encrypted payload i.e. tcp.Payload = tcp.Payload[5:] for further analysis?
Thanks
—
You are receiving this because you were mentioned.
Reply to this email directly, view it on GitHub <https://gist.github.com/290c994978b3832cbc2027f9f2656332?email_source=notifications&email_token=AAAZ7SZAYZYHHXDH3ONEZVTP2BSGZA5CNFSM4HXE6INKYY3PNVWWK3TUL52HS4DFVNDWS43UINXW23LFNZ2KUY3PNVWWK3TUL5UWJTQAFTQSC#gistcomment-2941217>, or mute the thread <https://github.com/notifications/unsubscribe-auth/AAAZ7S3V2KXHAINFKHI43K3P2BSGZANCNFSM4HXE6INA>.
Yes, if you have the TLS session key.
…
On Jun 11, 2019, at 11:09 PM, Saint Tukio @.***> wrote: I found this very useful @jrwren https://github.com/jrwren. Quick question, do you know how I can decrypt the encrypted payload i.e. tcp.Payload = tcp.Payload[5:] for further analysis? Thanks
Please do you have example how to achieve this -- I have logged the Session Key to file and trying all examples I can find online on how to decrypt but none seem to demonstrate using Session Key
. Any example will be more than invaluable indeed -- many thanks.
I have no examples.
I had to copy all of the go crypto/tls package into my own package to access the unexported PRF and use the unexported ciphersuites.
—
j
… On Jun 13, 2019, at 9:23 PM, Saint Tukio ***@***.***> wrote:
Yes, if you have the TLS session key.
… <x-msg://38/#>
On Jun 11, 2019, at 11:09 PM, Saint Tukio @.***> wrote: I found this very useful @jrwren <https://github.com/jrwren> https://github.com/jrwren <https://github.com/jrwren>. Quick question, do you know how I can decrypt the encrypted payload i.e. tcp.Payload = tcp.Payload[5:] for further analysis? Thanks
Please do you have example how to achieve this -- I have logged the Session Key to file and trying all examples I can find online on how to decrypt but none seem to demonstrate using Session Key. Any example will be more than invaluable really -- thanks.
—
You are receiving this because you were mentioned.
Reply to this email directly, view it on GitHub <https://gist.github.com/290c994978b3832cbc2027f9f2656332?email_source=notifications&email_token=AAAZ7S7LLWUQH5VIOBVGFZLP2LXIBA5CNFSM4HXE6INKYY3PNVWWK3TUL52HS4DFVNDWS43UINXW23LFNZ2KUY3PNVWWK3TUL5UWJTQAFTUTG#gistcomment-2943283>, or mute the thread <https://github.com/notifications/unsubscribe-auth/AAAZ7S6O3DRZER7J4MLQDJDP2LXIBANCNFSM4HXE6INA>.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment