Step 1: From you logs I have decoded the bucket/key
binary_to_term(<<131,109,0,0,0,9,85,114,108,84,111,83,99,97,110>>).
<<"UrlToScan">>
binary_to_term(<<131,109,0,0,0,61,104,116,116,112,37,51,97,37,50,102,37,50,102,119,119,119,46,97,110,104,114,105,46,110,101,116,37,50,102,37,51,102,102,101,101,100,37,51,100,114,115,115,50,37,50,54,97,109,112,37,51,98,112,37,51,100,55,51,49,49,48>>).
<<"http%3a%2f%2fwww.anhri.net%2f%3ffeed%3drss2%26amp%3bp%3d73110">>
Step 2: Please run the below snippits from riak attach to identify the owning partitions of this bucket/key
Bucket = <<"UrlToScan">>,
Key = <<"http%3a%2f%2fwww.anhri.net%2f%3ffeed%3drss2%26amp%3bp%3d73110">>,
BKey = {Bucket,Key},
{ok, Ring} = riak_core_ring_manager:get_my_ring(),
DocIdx = riak_core_util:chash_key(BKey),
BucketProps = riak_core_bucket:get_bucket(Bucket, Ring),
[NValue] = [Y || {X1, Y} <- BucketProps, n_val == X1],
UpNodes = riak_core_node_watcher:nodes(riak_kv),
Preflist2 = riak_core_apl:get_apl_ann(DocIdx, NValue, Ring, UpNodes),
Preflist = [{IndexNode, Type} || {IndexNode, Type} <- Preflist2].
Step 3: From the output of the above run the following on all N owners from riak attach inserting the appropriate partition ID depending on the node.
GS = fun(Pid) ->
{status, Pid, _Mod, Status} = sys:get_status(Pid),
Status2 = lists:flatten(Status),
Status3 = [L || {data, L} <- Status2],
Status4 = lists:flatten(Status3),
State = proplists:get_value("StateData", Status4),
State
end.
LR = fun(Idx) ->
{ok, Pid} = riak_core_vnode_manager:get_vnode_pid(Idx, riak_kv_vnode),
State = GS(Pid),
ModState = element(4, State),
case element(3,ModState) of
riak_kv_eleveldb_backend ->
LvlState = element(4, ModState),
element(2, LvlState);
riak_kv_multi_backend ->
MultiState = element(2,element(4, ModState)),
[State1] = [State0 || {<<"devices_eleveldb">>,riak_kv_eleveldb_backend,State0} <- MultiState],
element(2, State1);
_ ->
undefined
end
end.
Ref = LR(INSERT_PARTITIONID_HERE).
Bucket = <<"UrlToScan">>.
Key = <<"http%3a%2f%2fwww.anhri.net%2f%3ffeed%3drss2%26amp%3bp%3d73110">>.
{ok, O} = eleveldb:get(Ref,sext:encode({o,Bucket,Key}),[]).
binary_to_term(O).
The partition returning an error on the binary_to_term call is the corrupt copy.
Step 4: Only run the following on the node which contains the corrupt copy
eleveldb:delete(Ref,sext:encode({o,Bucket,Key}),[]).
Step 5: From any node, force read repair. This can be done through riak attach.
{ok, C} = riak:local_client().
C:get(Bucket,Key,[]).
Verify the repair was succesful by re-attaching to the node with the corrupt copy and re-running:
GS = fun(Pid) ->
{status, Pid, _Mod, Status} = sys:get_status(Pid),
Status2 = lists:flatten(Status),
Status3 = [L || {data, L} <- Status2],
Status4 = lists:flatten(Status3),
State = proplists:get_value("StateData", Status4),
State
end.
LR = fun(Idx) ->
{ok, Pid} = riak_core_vnode_manager:get_vnode_pid(Idx, riak_kv_vnode),
State = GS(Pid),
ModState = element(4, State),
case element(3,ModState) of
riak_kv_eleveldb_backend ->
LvlState = element(4, ModState),
element(2, LvlState);
riak_kv_multi_backend ->
MultiState = element(2,element(4, ModState)),
[State1] = [State0 || {<<"devices_eleveldb">>,riak_kv_eleveldb_backend,State0} <- MultiState],
element(2, State1);
_ ->
undefined
end
end.
Ref = LR(INSERT_PARTITIONID_HERE).
Bucket = <<"UrlToScan">>.
Key = <<"http%3a%2f%2fwww.anhri.net%2f%3ffeed%3drss2%26amp%3bp%3d73110">>.
{ok, O} = eleveldb:get(Ref,sext:encode({o,Bucket,Key}),[]).
binary_to_term(O).
The binary_to_term should no longer fail.