Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Patch for FNV1A_CH support in graphite-web 0.9.15
diff -u -r graphite_old/render/datalib.py graphite/render/datalib.py
--- graphite_old/render/datalib.py 2016-10-19 18:22:37.000000000 +0200
+++ graphite/render/datalib.py 2016-10-19 18:12:00.000000000 +0200
@@ -106,7 +106,7 @@
self.hosts = [ (server, instance) for (server, port, instance) in hosts ]
self.ports = dict( ((server, instance), port) for (server, port, instance) in hosts )
self.timeout = float(timeout)
- self.hash_ring = ConsistentHashRing(self.hosts)
+ self.hash_ring = ConsistentHashRing(self.hosts, hash_type=settings.CARBONLINK_HASHING_TYPE)
self.connections = {}
self.last_failure = {}
# Create a connection pool for each host
diff -u -r graphite_old/render/hashing.py graphite/render/hashing.py
--- graphite_old/render/hashing.py 2016-10-19 18:22:37.000000000 +0200
+++ graphite/render/hashing.py 2016-10-19 18:27:39.000000000 +0200
@@ -22,6 +22,24 @@
from itertools import chain
import bisect
+try:
+ import pyhash
+ hasher = pyhash.fnv1a_32()
+ def fnv32a(string, seed=0x811c9dc5):
+ return hasher(string, seed=seed)
+except ImportError:
+ def fnv32a(string, seed=0x811c9dc5):
+ """
+ FNV-1a Hash (http://isthe.com/chongo/tech/comp/fnv/) in Python.
+ Taken from https://gist.github.com/vaiorabbit/5670985
+ """
+ hval = seed
+ fnv_32_prime = 0x01000193
+ uint32_max = 2 ** 32
+ for s in string:
+ hval = hval ^ ord(s)
+ hval = (hval * fnv_32_prime) % uint32_max
+ return hval
def hashRequest(request):
# Normalize the request parameters so ensure we're deterministic
@@ -49,20 +67,28 @@
class ConsistentHashRing:
- def __init__(self, nodes, replica_count=100):
+ def __init__(self, nodes, replica_count=100, hash_type='carbon_ch'):
self.ring = []
+ self.hash_type = hash_type
self.replica_count = replica_count
for node in nodes:
self.add_node(node)
def compute_ring_position(self, key):
- big_hash = md5( str(key) ).hexdigest()
- small_hash = int(big_hash[:4], 16)
+ if self.hash_type == 'fnv1a_ch':
+ big_hash = ('%X' % int(fnv32a(str(key)))).lower()
+ small_hash = int(big_hash[:4], 16) ^ int(big_hash[4:], 16)
+ else:
+ big_hash = md5( str(key) ).hexdigest()
+ small_hash = int(big_hash[:4], 16)
return small_hash
def add_node(self, key):
for i in range(self.replica_count):
- replica_key = "%s:%d" % (key, i)
+ if self.hash_type == 'fnv1a_ch':
+ replica_key = "%d-%s" % (i, key[1])
+ else:
+ replica_key = "%s:%d" % (key, i)
position = self.compute_ring_position(replica_key)
entry = (position, key)
bisect.insort(self.ring, entry)
diff -u -r graphite_old/settings.py graphite/settings.py
--- graphite_old/settings.py 2016-10-19 18:22:37.000000000 +0200
+++ graphite/settings.py 2016-10-19 18:03:25.000000000 +0200
@@ -80,6 +80,7 @@
CARBONLINK_HOSTS = ["127.0.0.1:7002"]
CARBONLINK_TIMEOUT = 1.0
CARBONLINK_QUERY_BULK = False
+CARBONLINK_HASHING_TYPE = 'fnv1a_ch'
SMTP_SERVER = "localhost"
DOCUMENTATION_URL = "http://graphite.readthedocs.org/"
ALLOW_ANONYMOUS_CLI = True
@shulima

This comment has been minimized.

Copy link

@shulima shulima commented Apr 24, 2017

Note: if you intend to use this patch to get fnv1a_ch in your graphite 0.9.15 installation, change the big_hash calculation in compute_ring_position to
big_hash = ('%08X' % int(fnv32a(str(key)))).lower()
otherwise you'll end up with some mismatches.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment