angrycub

#Use pprof debug docker daemon

• Use pprof debug docker daemon
• pprof debug entrypoint
• Start docker daemon in debug mode
• Run socat to make docker sock available via tcp port
• Access debug url entrypoint
  • show global vars
  • get command line
  • run pprof on your client:

angrycub

-module(pinger).

-export([ping/1,
         ping/3,
         pinglist/4,
         pingall/0,
         pingall/1,
         pingall/2
]).

angrycub

#!/usr/bin/env escript
%% -*- erlang -*-

-include_lib("kernel/include/file.hrl").
-compile(export_all).
-define(LOG(S), io:format(S)).
-define(LOG(S,A), io:format(S,A)).

main(Dirs) ->
    CodePath = case os:getenv("RIAK_LIB") of

angrycub

Summary

LevelDB can become corrupted when bad things happen on the filesystem or in hardware. We push the I/O to the limits on heavily loaded Riak nodes so it is not uncommon to experience such failures. This one exhibits as a message Compaction error: Corruption: corrupted compressed block contents in the «data_root»/leveldb/«vnode»/LOG file.

Diagnosis

Steps that pin-point this issue

[root@prod-2163 /var/db/riak/leveldb]# find . -name "LOG" -exec grep -l 'Compaction error' {} \; 
./442446784738847563128068650529343492278651453440/LOG 

angrycub

PBStatusFun = fun() ->
                      VnodePids = [Pid || {_, Pid} <- riak_core_vnode_manager:all_index_pid(riak_kv_vnode)],
                      Links = [process_info(Pid, [links]) || Pid <- VnodePids],
                      WorkerPoolPids = [WPPid || [{links,[_, WPPid]}] <- Links],
                      WorkerPoolLinks = [process_info(Pid, [links]) || Pid <- WorkerPoolPids],
                      PoolboyPids = [PoolboyPid || [{links,[_, PoolboyPid]}] <- WorkerPoolLinks],
                      [poolboy:status(Pid) || Pid <- PoolboyPids]
              end.

PBStatusFun = fun(Index) ->

angrycub

% Purpose: I use this pre-commit hook to mark objects in a bucket as "dirty" with secondary indexing.
%          I then use a script to scrape out all dirty objects, do some processing, then save them with
%          "dirty_bin = false" as an index and the pre-commit hook erases the "dirty_bin" index.
%          So in essence it works as: `if dirty_bin = false; del dirty_bin; else dirty_bin = true; end`
% 
% To install this pre-commit hook (just like any Riak pre-commit hook in Erlang), you need to create an Erlang file and
% put it in your "basho-patches" directory. For me, on Ubuntu, this was "/usr/lib/riak/lib/basho-patches".
% Once there, you need to compile it to a .beam file. This was helped by using the Riak provided erlc compiler, 
% which, on my Ubuntu system, was at "/usr/lib/riak/erts-5.8.5/bin/erlc"
%