troubleshoot.txt

sudo perf record -F 49 -a -g -- sleep 120
sudo perf record -F 99 -g -p 27817

http://www.brendangregg.com/perf.html
http://www.brendangregg.com/linuxperf.html
http://www.brendangregg.com/FlameGraphs/cpuflamegraphs.html#Linux

export TERM=xterm; export PS1="\\w\$ ";
apk --no-cache add curl tcpdump vim

args: ["--endpoints.configprops.enabled=true"]
endpoints.autoconfig.enabled=true
endpoints.beans.enabled=true
endpoints.loggers.enabled=true
endpoints.configprops.enabled=true

apk search tcpdump
apk add tcpdump
apk del tcpdump

tcpdump -i eth0 udp port 12201 -vv -X
tcpdump -i eth0 udp port 12201 -vv

tcpdump for 
k exec -ti pod_name -- sh -c "apk add --no-cache tcpdump; tcpdump port 9200"
k exec -ti pod_name -- tcpdump 'port 80 and host not 10.20.36.1' -v
k exec -ti pod_name -- sh -c "apk add tcpdump; tcpdump -c 10 'port 80 and host not 10.20.36.1' -v"
k exec -ti pod_name -- sh -c "apk add tcpdump; tcpdump port 9411 and host not 10.20.85.1 -A"

k exec -ti pod_name -- tcpdump port 9411 -XX -w - | wireshark -k -i -
k exec -ti pod_name -- sh -c "tcpdump port 9411 -w - 2>/dev/null" | wireshark -k -i -

tcpdump 'port 80 and host not (10.2.15.4 or 10.2.42.4)' -vv
exclude prometheus

apk del tcpdump

k exec -ti user-history-service-6f47df869f-6lnqd  -- jcmd 1 Thread.print > user-history.thread.dump

jcmd 1 Thread.print
jcmd 1 VM.native_memory
jcmd 1 GC.class_histogram | head -n 100

lsof 1

JMX
-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Djava.rmi.server.hostname=127.0.0.1 -Dcom.sun.management.jmxremote.rmi.port=9606 -Dcom.sun.management.jmxremote.port=9606

remote debug
-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005

remove pod from service/deployment keeping it around
k label pod storage-service-c68bc8c8c-rtfp8 app=failing-storage-service --overwrite

after messing around
rm -rf /var/lib/apt/lists/*

zipkin force traces to export
spring.zipkin.enabled=false
spring.sleuth.sampler.percentage=1
X-B3-Flags: 1

tcpdump to file AND display
tcpdump -w - | tee somefile | tcpdump -r -
-w - tells tcpdump to write binary data to stdout
tee writes that binary data to a file AND to its own stdout
-r - tells the second tcpdump to get its data from its stdin

page cache view
http://code.google.com/p/linux-ftools/
fincore - page cache view

cronjob trigger:
kubectl create job --from=cronjob/ratings-report cron-ratings-report-manual-run
"*/1 * * * *"
kubectl --v 9

docker stopped container access disc
docker commit CONTAINER NEWIMAGENAME
docker run -ti --entrypoint /bin/bash NEWIMAGENAME


dstat
echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories
apk add --no-cache dstat@testing
apk add --no-cache perf@testing
dstat -lvrn 10

fio - IO performance

TCP Flag	tcpdump Flag	Meaning
SYN	S	Syn packet, a session establishment request.
ACK	A	Ack packet, acknowledge sender’s data.
FIN	F	Finish flag, indication of termination.
RESET	R	Reset, indication of immediate abort of conn.
PUSH	P	Push, immediate push of data from sender.
URGENT	U	Urgent, takes precedence over other data.
NONE	
A dot .
Placeholder, usually used for ACK.

the namespace is made available to pod as part of the service account secret and automatically detected by the client
/var/run/secrets/kubernetes.io/serviceaccount/namespace

https://www.digitalocean.com/community/tutorials/how-to-use-journalctl-to-view-and-manipulate-systemd-logs
journalctl -u mongod.service -f
-o json
-o json-pretty
-n 20
--utc
-b - current boot
journalctl --list-boots
journalctl --since "2015-01-10" --until "2015-01-11 03:00"
journalctl --since 09:00 --until "1 hour ago"
journalctl _PID=8088
man systemd.journal-fields
journalctl -F _GID - show available values

network
nmcli d wifi


top services 
curl -XPOST 'http://es-backend-stg.preprod-local.ch:80/jaeger-span-2018-02-12/_search' 

top applications:
{
 "size": 0,
 "aggs": {
  "uniq_host": {
  "terms": {
  "field": "process.serviceName",
  "size": 500
  }
  }
 }
}

top operations for an app:
{
 "size": 0,
 "aggs": {
  "uniq_host": {
  "terms": {
  "field": "operationName",
  "size": 500
  }
  }
 },
 "query": {
  "term": { "process.serviceName": "freecall" }
  }
}