Skip to content

Instantly share code, notes, and snippets.

@felixbarny

felixbarny/data-model.txt

Last active May 13, 2021 06:47
Show Gist options
  • Save felixbarny/901cb7ab90dd0c5cbfa769cd997df872 to your computer and use it in GitHub Desktop.
Save felixbarny/901cb7ab90dd0c5cbfa769cd997df872 to your computer and use it in GitHub Desktop.
Download ZIP
Elasticsearch profiling
Raw
data-model.txt
DELETE /profile?ignore_unavailable=true
PUT /profile
{
"settings": {
"index": {
"codec": "best_compression"
}
},
"mappings": {
"properties": {
"@timestamp": { "type": "date" },
"service": {
"properties": {
"name": { "type": "keyword" }
}
},
"profile": {
"properties": {
"levels": { "type": "integer", "index": false },
"samples": { "type": "long", "index": false },
"signatures": { "type": "integer", "index": false },
"symbols": { "type": "keyword", "index": false }
}
}
}
}
}
GET /profile/_mapping
#
# 2 [baz]
# 1 [bar]
# 0 [foo ][qux]
POST /profile/_doc?refresh
{
"@timestamp": "2021-05-10T00:00:00Z",
"service": {
"name": "testapp"
},
"profile": {
"levels": [ 0, 1, 2, 0],
"samples": [10, 5, 5, 5],
"signatures": [ 0, 1, 2, 3],
"symbols": ["foo", "bar", "baz", "qux"]
}
}
#
# 2 [baz]
# 1 [bar][bar]
# 0 [foo][qux ]
POST /profile/_doc?refresh
{
"@timestamp": "2021-05-10T00:00:10Z",
"service": {
"name": "testapp"
},
"profile": {
"levels": [0, 1, 2, 0, 1],
"samples": [5, 5, 5, 10, 5],
"signatures": [0, 1, 2, 3, 1],
"symbols": ["foo", "bar", "baz", "qux"]
}
}
POST /profile/_search
Raw
merge-tree-aggregation.txt
#
# 2 [baz ]
# 1 [bar ] [bar]
# 0 [foo ][qux ]
# "profile": {
# "levels": [ 0, 1, 2, 0, 1],
# "samples": [15, 10, 10, 15, 5],
# "signatures": [ 0, 1, 2, 3, 1],
# "symbols": ["foo", "bar", "baz", "qux"]
# }
POST /profile/_search
{
"query": {
"match_all": {}
},
"size": 0,
"aggs": {
"profile_agg": {
"scripted_metric": {
"init_script": "state.profiles = []",
"map_script": """
state.profiles.add(params._source["profile"])
""",
"combine_script": """
void addFrame(def flatTree, int level, String signature, def frame) {
flatTree["levels"].add(level);
flatTree["samples"].add(frame["samples"]);
flatTree["signatures"].add(symbolize(flatTree["symbols"], signature));
for (entry in frame["children"].entrySet()) {
addFrame(flatTree, level + 1, entry.key, entry.value);
}
}
int symbolize(def symbols, String signature) {
if (symbols["symbolToId"].containsKey(signature)) {
return symbols["symbolToId"][signature];
} else {
int symbolId = symbols["symbolToId"].size();
symbols["symbolToId"].put(signature, symbolId);
symbols["symbols"].add(signature);
return symbolId;
}
}
void merge(def mergedTree, def profile) {
Deque stack = new ArrayDeque();
stack.push(mergedTree);
for (int i = 0, levelsLength = profile.levels.length; i < levelsLength; i++) {
int level = (int) profile.levels[i] + 1;
while (level < stack.size()) {
stack.pop();
}
def parent = stack.peek();
long samples = profile.samples[i];
String signature = profile.symbols[(int) profile.signatures[i]];
def child = parent["children"][signature];
if (child == null) {
child = [
"samples": samples,
"children": [:]
];
parent["children"].put(signature, child);
} else {
child["samples"] = child["samples"] + samples;
}
stack.push(child);
}
}
def mergedTree = ["children": [:]];
for (profile in state.profiles) {
merge(mergedTree, profile);
}
def flatTree = [
"levels": [],
"samples": [],
"signatures": [],
"symbols": [
"symbols": [],
"symbolToId": [:]
]
];
for (entry in mergedTree["children"].entrySet()) {
addFrame(flatTree, 0, entry.key, entry.value);
}
flatTree["symbols"] = flatTree["symbols"]["symbols"];
return flatTree;
""",
"reduce_script": """
void addFrame(def flatTree, int level, String signature, def frame) {
flatTree["levels"].add(level);
flatTree["samples"].add(frame["samples"]);
flatTree["signatures"].add(symbolize(flatTree["symbols"], signature));
for (entry in frame["children"].entrySet()) {
addFrame(flatTree, level + 1, entry.key, entry.value);
}
}
int symbolize(def symbols, String signature) {
if (symbols["symbolToId"].containsKey(signature)) {
return symbols["symbolToId"][signature];
} else {
int symbolId = symbols["symbolToId"].size();
symbols["symbolToId"].put(signature, symbolId);
symbols["symbols"].add(signature);
return symbolId;
}
}
void merge(def mergedTree, def profile) {
Deque stack = new ArrayDeque();
stack.push(mergedTree);
for (int i = 0, levelsLength = profile.levels.length; i < levelsLength; i++) {
int level = (int) profile.levels[i] + 1;
while (level < stack.size()) {
stack.pop();
}
def parent = stack.peek();
long samples = profile.samples[i];
String signature = profile.symbols[(int) profile.signatures[i]];
def child = parent["children"][signature];
if (child == null) {
child = [
"samples": samples,
"children": [:]
];
parent["children"].put(signature, child);
} else {
child["samples"] = child["samples"] + samples;
}
stack.push(child);
}
}
def mergedTree = ["children": [:]];
for (profile in states) {
merge(mergedTree, profile);
}
def flatTree = [
"levels": [],
"samples": [],
"signatures": [],
"symbols": [
"symbols": [],
"symbolToId": [:]
]
];
for (entry in mergedTree["children"].entrySet()) {
addFrame(flatTree, 0, entry.key, entry.value);
}
flatTree["symbols"] = flatTree["symbols"]["symbols"];
return flatTree;
"""
}
}
}
}
Raw
transforms.txt
DELETE _transform/profile-1m?force=true
DELETE /profile-1m?ignore_unavailable=true
PUT /profile-1m
{
"settings": {
"index": {
"codec": "best_compression"
}
},
"mappings": {
"properties": {
"@timestamp": { "type": "date" },
"service": {
"properties": {
"name": { "type": "keyword" }
}
},
"profile": {
"properties": {
"levels": { "type": "integer", "index": false },
"samples": { "type": "long", "index": false },
"signatures": { "type": "integer", "index": false },
"symbols": { "type": "keyword", "index": false }
}
}
}
}
}
DELETE _transform/profile-1m?force=true
PUT _transform/profile-1m
{
"source": {
"index": "profile"
},
"dest": {
"index": "profile-1m"
},
"frequency": "5s",
"sync": {
"time": {
"field": "@timestamp",
"delay": "1m"
}
},
"pivot": {
"group_by": {
"service.name": {
"terms": {
"field": "service.name"
}
},
"@timestamp": {
"date_histogram": {
"field": "@timestamp",
"fixed_interval": "1m"
}
}
},
"aggregations": {
"profile_agg": {
"scripted_metric": {
"init_script": "state.profiles = []",
"map_script": """
state.profiles.add(params._source["profile"])
""",
"combine_script": """
void addFrame(def flatTree, int level, String signature, def frame) {
flatTree["levels"].add(level);
flatTree["samples"].add(frame["samples"]);
flatTree["signatures"].add(symbolize(flatTree["symbols"], signature));
for (entry in frame["children"].entrySet()) {
addFrame(flatTree, level + 1, entry.key, entry.value);
}
}
int symbolize(def symbols, String signature) {
if (symbols["symbolToId"].containsKey(signature)) {
return symbols["symbolToId"][signature];
} else {
int symbolId = symbols["symbolToId"].size();
symbols["symbolToId"].put(signature, symbolId);
symbols["symbols"].add(signature);
return symbolId;
}
}
void merge(def mergedTree, def profile) {
Deque stack = new ArrayDeque();
stack.push(mergedTree);
if (profile == null) throw new RuntimeException(profile);
for (int i = 0, levelsLength = profile.levels.length; i < levelsLength; i++) {
int level = (int) profile.levels[i] + 1;
while (level < stack.size()) {
stack.pop();
}
def parent = stack.peek();
long samples = profile.samples[i];
String signature = profile.symbols[(int) profile.signatures[i]];
def child = parent["children"][signature];
if (child == null) {
child = [
"samples": samples,
"children": [:]
];
parent["children"].put(signature, child);
} else {
child["samples"] = child["samples"] + samples;
}
stack.push(child);
}
}
def mergedTree = ["children": [:]];
for (profile in state.profiles) {
merge(mergedTree, profile);
}
def flatTree = [
"levels": [],
"samples": [],
"signatures": [],
"symbols": [
"symbols": [],
"symbolToId": [:]
]
];
for (entry in mergedTree["children"].entrySet()) {
addFrame(flatTree, 0, entry.key, entry.value);
}
flatTree["symbols"] = flatTree["symbols"]["symbols"];
return flatTree;
""",
"reduce_script": """
void addFrame(def flatTree, int level, String signature, def frame) {
flatTree["levels"].add(level);
flatTree["samples"].add(frame["samples"]);
flatTree["signatures"].add(symbolize(flatTree["symbols"], signature));
for (entry in frame["children"].entrySet()) {
addFrame(flatTree, level + 1, entry.key, entry.value);
}
}
int symbolize(def symbols, String signature) {
if (symbols["symbolToId"].containsKey(signature)) {
return symbols["symbolToId"][signature];
} else {
int symbolId = symbols["symbolToId"].size();
symbols["symbolToId"].put(signature, symbolId);
symbols["symbols"].add(signature);
return symbolId;
}
}
void merge(def mergedTree, def profile) {
Deque stack = new ArrayDeque();
stack.push(mergedTree);
for (int i = 0, levelsLength = profile.levels.length; i < levelsLength; i++) {
int level = (int) profile.levels[i] + 1;
while (level < stack.size()) {
stack.pop();
}
def parent = stack.peek();
long samples = profile.samples[i];
String signature = profile.symbols[(int) profile.signatures[i]];
def child = parent["children"][signature];
if (child == null) {
child = [
"samples": samples,
"children": [:]
];
parent["children"].put(signature, child);
} else {
child["samples"] = child["samples"] + samples;
}
stack.push(child);
}
}
def mergedTree = ["children": [:]];
for (profile in states) {
if (profile != null) merge(mergedTree, profile);
}
def flatTree = [
"levels": [],
"samples": [],
"signatures": [],
"symbols": [
"symbols": [],
"symbolToId": [:]
]
];
for (entry in mergedTree["children"].entrySet()) {
addFrame(flatTree, 0, entry.key, entry.value);
}
flatTree["symbols"] = flatTree["symbols"]["symbols"];
return flatTree;
"""
}
}
}
}
}
POST _transform/profile-1m/_start
GET _transform/profile-1m/_stats
POST profile-1m/_search
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment