Skip to content

Instantly share code, notes, and snippets.

@ndimiduk

ndimiduk/demo.html

Created May 21, 2021 20:23
Show Gist options
  • Save ndimiduk/2305c1ffd4e02b96e56bfc04db789740 to your computer and use it in GitHub Desktop.
Save ndimiduk/2305c1ffd4e02b96e56bfc04db789740 to your computer and use it in GitHub Desktop.
Download ZIP
Demoing Region Visualizer
Raw
demo.html
<!DOCTYPE html>
<!--
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-->
<html>
<head>
<title>Embedding Vega-Lite</title>
<script src="https://cdn.jsdelivr.net/npm/vega@5.19.1/build/vega.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/vega-lite@5.0.0/build/vega-lite.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/vega-embed@6.15.1/build/vega-embed.min.js"></script>
</head>
<body>
<div id="vis" style='width:80%;'></div>
<script type="text/javascript">
vega.expressionFunction('values', function(obj) { return Object.values(obj); });
var yourVlSpec = {
$schema: 'https://vega.github.io/schema/vega-lite/v5.json',
description: 'Total `storefileSize` per Region Server',
data: {
name: 'region_info',
//url: 'http://localhost:16010/api/v1/admin/cluster_metrics/live_servers',
url: './server_metrics.json',
format: { type: 'json', property: 'data' }
},
transform: [
{ calculate: "split(datum.server_name.servername, ',', 1)[0]", as: 'server_name' },
{ calculate: 'values(datum.region_status)', as: 'region_status' },
{ flatten: ['region_status'] },
{ calculate: "split(datum.region_status.name, ',', 1)[0]", as: 'table_name' },
{ calculate: "split(datum.region_status.name, ',', 3)[2]", as: 'region_id' },
],
vconcat: [{
title: {
text: 'Total `storefileSize` per Region Server',
align: 'center'
},
height: 250,
width: 'container',
mark: { type: 'bar', tooltip: true },
params: [{
name: 'legend_selection',
select: { type: 'point', fields: ['table_name'] },
bind: 'legend'
}],
encoding: {
x: {
title:'Region Server',
field: 'server_name',
type: 'nominal',
axis: {
labelAngle: 90,
labelOverlap: 'greedy'
},
scale: { domain: { param: 'brush' } },
sort: '-y'
},
y: {
title: 'Total StoreFileSize',
field: 'region_status.store_file_size',
type: 'quantitative',
aggregate: 'sum',
format: '.4s',
axis: { format: '.2s' },
scale: { domainMin: 0 }
},
color: {
field: 'table_name',
type: 'nominal',
title: 'Table'
},
opacity: {
condition: { param: 'legend_selection', value: 0.9 },
value: 0.2
}
}
}, {
mark: 'bar',
height: 50,
width: 'container',
params: [{
name: 'brush',
select: { type: 'interval', encodings: ['x'] }
}],
encoding: {
x: {
field: 'server_name',
type: 'nominal',
axis: null,
sort: '-y'
},
y: {
field: 'region_status.store_file_size',
aggregate: 'sum',
axis: null
},
color: {
field: 'table_name',
title: 'Table'
}
}
}]
};
vegaEmbed('#vis', yourVlSpec)
.then(function(result) {
//result.view.logger(vega.Debug);
console.log(result.view.data('region_info'));
console.log(result.view.width());
})
.catch(console.Error);
</script>
</body>
</html>
Raw
dump_cluster_status.jsh
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.FileOutputStream;
import java.io.OutputStreamWriter;
import java.lang.reflect.Type;
import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.EnumSet;
import java.util.Map;
import org.apache.hadoop.hbase.ClusterMetrics;
import org.apache.hadoop.hbase.ClusterMetrics.Option;
import org.apache.hadoop.hbase.Size;
import org.apache.hadoop.hbase.client.AsyncAdmin;
import org.apache.hadoop.hbase.client.AsyncConnection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap;
import org.apache.hbase.thirdparty.com.google.gson.FieldNamingPolicy;
import org.apache.hbase.thirdparty.com.google.gson.Gson;
import org.apache.hbase.thirdparty.com.google.gson.GsonBuilder;
import org.apache.hbase.thirdparty.com.google.gson.JsonElement;
import org.apache.hbase.thirdparty.com.google.gson.JsonPrimitive;
import org.apache.hbase.thirdparty.com.google.gson.JsonSerializationContext;
import org.apache.hbase.thirdparty.com.google.gson.JsonSerializer;
class ByteArraySerializer implements JsonSerializer<byte[]> {
@Override
public JsonElement serialize(byte[] src, Type typeOfSrc, JsonSerializationContext context) {
return new JsonPrimitive(Bytes.toString(src));
}
}
class SizeAsBytesSerializer implements JsonSerializer<Size> {
@Override
public JsonElement serialize(Size src, Type typeOfSrc, JsonSerializationContext context) {
return new JsonPrimitive(src.get(Size.Unit.BYTE));
}
}
{
final Gson gson = new GsonBuilder()
.setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
.enableComplexMapKeySerialization()
.registerTypeAdapter(byte[].class, new ByteArraySerializer())
.registerTypeAdapter(Size.class, new SizeAsBytesSerializer())
.setPrettyPrinting()
.create();
Collection serverMetrics;
try (AsyncConnection conn = ConnectionFactory.createAsyncConnection().get()) {
AsyncAdmin admin = conn.getAdmin();
serverMetrics = admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
.thenApply(ClusterMetrics::getLiveServerMetrics)
.thenApply(Map::values)
.get();
}
try (FileOutputStream outputStream = new FileOutputStream("server_metrics.json");
OutputStreamWriter writer = new OutputStreamWriter(outputStream, StandardCharsets.UTF_8)) {
gson.toJson(ImmutableMap.of("data", serverMetrics), writer);
writer.flush();
}
}
/exit
Raw
instructions.md

Usage

To demo this visualization, you'll need a snapshot of your cluster's ClusterMetrics, specifically the "live servers" list of ServerMetrics. To collect that, use the file dump_cluster_status.jsh` with any HBase 2.3+ client and JDK11.

$ CLASSPATH="$(env JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 hbase classpath)" /usr/lib/jvm/java-11-openjdk-amd64/bin/jshell ./dump_cluster_status.jsh

This produces the file server_metrics.json in the local directory.

Now run the demo. To do so, place server_metrics.json and demo.html in the same directory. Launch a webserver hosting that directory, for instance, with python:

$ python3 -m http.server
Serving HTTP on :: port 8000 (http://[::]:8000/) ...

Now browse to http://localhost:8000/demo.html. If all is working, you'll see the visualization.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment