Skip to content

Instantly share code, notes, and snippets.

@richcollier
Created August 26, 2022 21:49
Show Gist options
  • Save richcollier/5643e649a2816ed317d0caf3917263b8 to your computer and use it in GitHub Desktop.
Save richcollier/5643e649a2816ed317d0caf3917263b8 to your computer and use it in GitHub Desktop.
compare_shard_primary_and_replica
POST _watcher/watch/_execute
{
"watch": {
"trigger": {
"schedule": {
"interval": "1d"
}
},
"input": {
"http": {
"request": {
"host": "de22790622f1457db955f0db15cbf8e6.eastus2.azure.elastic-cloud.com",
"port": 9243,
"scheme": "https",
"path": "/_cat/shards/kibana_sample_*",
"params": {
"format": "json",
"bytes": "b",
"human": "true",
"h": "index,shard,prirep,state,docs,store,maxSeqNo,gmto,iif,sync_id,node"
},
"headers": {},
"auth": {
"basic": {
"username": "elastic",
"password": "xxxxxxxx"
}
}
}
}
},
"condition": {
"script": """
def indices = new HashMap();
for (def data : ctx.payload.data) {
String key = data.index + "-" + data.shard;
def info = indices.get(key);
if (info == null) {
info = new HashMap();
indices.put(key, info);
}
if (data.prirep == "p") {
info.put("pdocs",data.docs);
info.put("pstore",data.store);
info.put("pnode",data.node);
} else if (data.prirep == "r") {
info.put("rdocs",data.docs);
info.put("rstore",data.store);
info.put("rnode",data.node);
}
}
Iterator entries = indices.entrySet().iterator();
while (entries.hasNext()) {
Map.Entry entry = (Map.Entry) entries.next();
String index_shard = (String)entry.getKey();
HashMap info = (HashMap)entry.getValue();
Iterator info_details = info.entrySet().iterator();
int pstore = Integer.parseInt(info.get("pstore"));
int rstore = Integer.parseInt(info.get("rstore"));
int pdocs = Integer.parseInt(info.get("pdocs"));
int rdocs = Integer.parseInt(info.get("rdocs"));
if (pstore != rstore){
return true;
}
}
"""
},
"actions": {
"displayResults": {
"transform": {
"script": """
def indices = new HashMap();
for (def data : ctx.payload.data) {
String key = data.index + "-" + data.shard;
def info = indices.get(key);
if (info == null) {
info = new HashMap();
indices.put(key, info);
}
if (data.prirep == "p") {
info.put("pdocs",data.docs);
info.put("pstore",data.store);
info.put("pnode",data.node);
} else if (data.prirep == "r") {
info.put("rdocs",data.docs);
info.put("rstore",data.store);
info.put("rnode",data.node);
}
}
def violators = new HashMap();
Iterator entries = indices.entrySet().iterator();
while (entries.hasNext()) {
def values = [];
Map.Entry entry = (Map.Entry) entries.next();
String index_shard = (String)entry.getKey();
HashMap info = (HashMap)entry.getValue();
Iterator info_details = info.entrySet().iterator();
int pstore = Integer.parseInt(info.get("pstore"));
int rstore = Integer.parseInt(info.get("rstore"));
int pdocs = Integer.parseInt(info.get("pdocs"));
int rdocs = Integer.parseInt(info.get("rdocs"));
String pnode = info.get("pnode");
String rnode = info.get("rnode");
if (pstore != rstore){
values.add("pstore="+pstore);
values.add("rstore="+rstore);
values.add("pdocs="+pdocs);
values.add("rdocs="+rdocs);
values.add("pnode="+pnode);
values.add("rnode="+rnode);
}
violators.put(index_shard,values);
}
return violators;
"""
},
"logging": {
"text": "list of violators are: {{ctx.payload}}"
}
}
}
}
}
@richcollier
Copy link
Author

I compare store on primary vs. replica (could do docs alternatively). Result looks like:

      "actions": [
        {
          "id": "displayResults",
          "type": "logging",
          "status": "success",
          "transform": {
            "type": "script",
            "status": "success",
            "payload": {
              "kibana_sample_data_flights-0": {
                "pdocs": "13059",
                "pstore": "6210589",
                "rdocs": "13059",
                "rstore": "6248284",
                "rnode": "instance-0000000001",
                "pnode": "instance-0000000000"
              },
              "kibana_sample_data_logs-0": {
                "pdocs": "14074",
                "rdocs": "14074",
                "pstore": "9379318",
                "rstore": "9277925",
                "rnode": "instance-0000000003",
                "pnode": "instance-0000000001"
              },
              "kibana_sample_data_ecommerce-0": {
                "pdocs": "4675",
                "rdocs": "4675",
                "pstore": "4559490",
                "rstore": "4707890",
                "rnode": "instance-0000000003",
                "pnode": "instance-0000000000"
              }
            }
          },
          "logging": {
            "logged_text": "list is {kibana_sample_data_flights-0={pdocs=13059, pstore=6210589, rdocs=13059, rstore=6248284, rnode=instance-0000000001, pnode=instance-0000000000}, kibana_sample_data_logs-0={pdocs=14074, rdocs=14074, pstore=9379318, rstore=9277925, rnode=instance-0000000003, pnode=instance-0000000001}, kibana_sample_data_ecommerce-0={pdocs=4675, rdocs=4675, pstore=4559490, rstore=4707890, rnode=instance-0000000003, pnode=instance-0000000000}}"
          }
        }
      ]

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment