Mucid/README.md

## README.md

      
    Raw
  

              README.md
            
          
    #This extension enhance higgs cluster stability and friendly to home/nat user
sudo pip install -U requests dnspython
sudo cp higgs_plus.py /usr/sbin/higgs_plus

sudo chmod 755 /usr/sbin/higgs_plus
sudo /usr/sbin/higgs_plus -cc

#Then you should modify /etc/higgs/rait.conf

#peers = "https://www.catofes.com/higgs.hcl"

#to

#peers = "/etc/higgs/higgs_addr.hcl"
sudo cp higgs_plus.service /etc/systemd/system/higgs_plus.service

sudo systemctl daemon-reload

sudo systemctl enable higgs_plus

sudo systemctl restart higgs_plus

  
## higgs_plus.py
#!/usr/bin/env python
# coding=UTF-8
"""
Created on Dec 24, 2020

@author: mucid
@email: muciwd@gmail.com
@blog: brilliant.run
"""


# Support Python2 And Python3 !


# Require
# pip install -U requests dnspython
# Optional
# pip install -U pyhcl


PeersURL = "https://www.catofes.com/higgs.hcl"


from datetime import datetime
import dns.resolver
import collections
import argparse
import requests
import time
import sys
import os


gContext = collections.namedtuple("gContext", ["py_ver", "version", "debug", "etag"])
gContext.py_ver = sys.version_info.major
gContext.version = "1.0.0"
gContext.debug = False
gContext.etag = ""


def sPrint(message):
    cTime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    print("%s\t%s" % (cTime, message))
    sys.stdout.flush()
    return


def createDNSResolver():
    resolver = dns.resolver.Resolver()
    resolver.timeout = 9
    resolver.lifetime = 3
    return resolver


def queryDNSRecord(dnsResolver, domain, version, retry=10):
    answers = None
    addrGroup = []
    for _ in range(retry):
        if version == 4:
            try:
                if gContext.py_ver > 2:
                    answers = dnsResolver.resolve(domain, "A")
                else:
                    answers = dnsResolver.query(domain, "A")
            except:
                answers = None
        elif version == 6:
            try:
                if gContext.py_ver > 2:
                    answers = dnsResolver.resolve(domain, "AAAA")
                else:
                    answers = dnsResolver.query(domain, "AAAA")
            except:
                answers = None
        if answers:
            for addr in answers:
                addrGroup.append(addr.address)
            if addrGroup:
                break
        else:
            time.sleep(1.5)
    if addrGroup:
        addrGroup.sort()
        return (domain, addrGroup[0])
    else:
        return (domain, "")
    return addrGroup


def fetchPeersHcl():
    try:
        etag = requests.head(PeersURL).headers["Etag"]
        if not gContext.etag:
            gContext.etag = etag
            if gContext.debug:
                sPrint("init data for %s" % (PeersURL,))
        elif gContext.etag == etag:
            if gContext.debug:
                sPrint("no change for %s" % (PeersURL,))
            return []
    except:
        return []
    import hcl

    try:
        if gContext.py_ver > 2:
            peersData = requests.get(PeersURL, timeout=10).content.decode()
        else:
            peersData = requests.get(PeersURL, timeout=10).content
    except Exception as e:
        if gContext.debug:
            sPrint(e)
        return []
    peersGroup = hcl.loads(peersData)
    if not peersGroup:
        return []
    dnsGroup = []
    for peer in peersGroup["peers"]:
        endpoint = peer.get("endpoint", {})
        if endpoint:
            dnsName = (endpoint["address"], endpoint["address_family"])
            if dnsName[0]:
                dnsGroup.append(dnsName)
    dnsGroup.sort(key=lambda x: "%s_%s" % (x[0], x[1]))
    return dnsGroup


def fetchPeersFake():
    try:
        etag = requests.head(PeersURL).headers["Etag"]
        if not gContext.etag:
            gContext.etag = etag
            if gContext.debug:
                sPrint("init data for %s" % (PeersURL,))
        elif gContext.etag == etag:
            if gContext.debug:
                sPrint("no change for %s" % (PeersURL,))
            return []
    except:
        return []
    try:
        if gContext.py_ver > 2:
            peersData = requests.get(PeersURL, timeout=10).content.decode()
        else:
            peersData = requests.get(PeersURL, timeout=10).content
    except Exception as e:
        if gContext.debug:
            sPrint(e)
        return []
    with open("/etc/higgs/higgs_name.hcl", "w") as fp:
        fp.write(peersData)
        fp.flush()
    dnsGroup = []
    pd = peersData.replace("\r", "").split("\n")
    t = {
        "0": "",
        "1": "",
    }
    st = []
    for i in pd:
        i = i.replace(" ", "")
        if i == "endpoint{":
            st.append(0)
        elif i == "}":
            if st:
                if t["0"]:
                    dnsGroup.append((t["0"], t["1"]))
                st.pop()
        if st:
            i = i.split("=")
            if i[0] == "address":
                t["0"] = i[1].replace('"', "")
            elif i[0] == "address_family":
                t["1"] = i[1].replace('"', "")
    dnsGroup.sort(key=lambda x: "%s_%s" % (x[0], x[1]))
    return dnsGroup


def checkPeers(dnsGroup):
    sp = time.time()
    dnsRecords = {}
    dnsResolver = createDNSResolver()
    for dnsName in dnsGroup:
        if dnsName[1] == "ip4":
            ret = queryDNSRecord(dnsResolver, dnsName[0], 4)
        elif dnsName[1] == "ip6":
            ret = queryDNSRecord(dnsResolver, dnsName[0], 6)
        dnsRecords["%s_%s" % (dnsName[0], dnsName[1])] = ret[1]
        time.sleep(0.05)
    ep = time.time()
    if gContext.debug:
        sPrint("DNS Query Time Spend: %.3fs" % (ep - sp,))
    return dnsRecords


def updatePeers(dnsRecords):
    with open("/etc/higgs/higgs_name.hcl", "r") as fp:
        pd = fp.readlines()
    t = {
        "0": "",
        "1": "",
    }
    st = []
    for n1, i in enumerate(pd):
        i = i.replace("\r", "").replace("\n", "").replace(" ", "")
        if i == "endpoint{":
            st.append(0)
        elif i == "}":
            if st:
                if t["0"]:
                    n2 = n1 - 1
                    while True:
                        j = pd[n2].replace("\r", "").replace("\n", "").replace(" ", "")
                        if j.find("address=") > -1:
                            pd[n2] = pd[n2].replace(
                                t["0"], dnsRecords["%s_%s" % (t["0"], t["1"])]
                            )
                            break
                        else:
                            n2 -= 1
                st.pop()
        if st:
            i = i.split("=")
            if i[0] == "address":
                t["0"] = i[1].replace('"', "")
            elif i[0] == "address_family":
                t["1"] = i[1].replace('"', "")
    with open("/etc/higgs/higgs_addr.hcl", "w") as fp:
        fp.write("".join(pd))
        fp.flush()
    return


def doSystemAction(systemAction):
    try:
        command = "bash -c '%s'" % (systemAction,)
        if gContext.debug:
            sPrint(command)
        os.system(command)
    except Exception as e:
        sPrint(e)
    return


def higgsPlusVisor(systemAction, heartBeat, createConfig=False, IsStableNode=False):
    while True:
        # lastDnsRecords=checkPeers(fetchPeersHcl())
        lastDnsRecords = checkPeers(fetchPeersFake())
        if lastDnsRecords:
            break
        else:
            time.sleep(heartBeat)
    shouldRefresh = False
    updatePeers(lastDnsRecords)
    if createConfig:
        sPrint("Finished Create Config At %s" % ("/etc/higgs/higgs_addr.hcl",))
        return
    for n, i in enumerate(lastDnsRecords.items()):
        sPrint("%-3d %s --> %s" % (n + 1, i[0], i[1]))
    sPrint("Finished Initinal Higgs Plus System....")
    while True:
        time.sleep(heartBeat)
        newDnsRecords = checkPeers(fetchPeersFake())
        if len(newDnsRecords) == 0:
            shouldRefresh = False
        elif len(lastDnsRecords) == len(newDnsRecords):
            for name, addr in newDnsRecords.items():
                if lastDnsRecords[name] != addr:
                    if (addr) and (not IsStableNode):
                        shouldRefresh = True
                        sPrint("Detect DNS Record Change %s --> %s" % (name, addr))
        elif len(lastDnsRecords) != len(newDnsRecords):
            shouldRefresh = True
            sPrint("Detect Node Change * <--> *")
        if shouldRefresh:
            shouldRefresh = False
            lastDnsRecords = newDnsRecords
            updatePeers(lastDnsRecords)
            doSystemAction(systemAction)
            sPrint("Finished Refresh Higgs Service !")
            time.sleep(heartBeat * 3)
    return


def main():
    opt = argparse.ArgumentParser()
    opt.add_argument("-d", "--debug", action="store_true", help="enable debug")
    opt.add_argument(
        "-cc", "--CreateConfig", action="store_true", help="Create Config For Initinal"
    )
    opt.add_argument(
        "-is",
        "--IsStableNode",
        action="store_true",
        help="Is Stable Node (for vps optimize, home/nat needn't, but not suggest enable)",
    )
    opt.add_argument(
        "-sa",
        "--SystemAction",
        type=str,
        default="echo Do Nothing....",
        help="Refresh Command To Use('systemctl reload higgs')",
    )
    opt.add_argument(
        "-hb",
        "--HeartBeat",
        type=int,
        default=30,
        help="Refresh Check Interval(30 seconds)",
    )
    param = opt.parse_args()
    if param.debug:
        gContext.debug = True
    try:
        higgsPlusVisor(
            param.SystemAction, param.HeartBeat, param.CreateConfig, param.IsStableNode
        )
    except Exception as e:
        sPrint("\n%s" % e)
    return


if __name__ == "__main__":
    main()

## higgs_plus.service
[Unit]
Description=higgs plus
After=higgs.service
Wants=higgs.service

[Service]
Restart=always
ExecStart=/usr/bin/python /usr/sbin/higgs_plus -sa 'systemctl reload higgs'

[Install]
WantedBy=multi-user.target
	#!/usr/bin/env python
	# coding=UTF-8
	"""
	Created on Dec 24, 2020

	@author: mucid
	@email: muciwd@gmail.com
	@blog: brilliant.run
	"""


	# Support Python2 And Python3 !


	# Require
	# pip install -U requests dnspython
	# Optional
	# pip install -U pyhcl


	PeersURL = "https://www.catofes.com/higgs.hcl"


	from datetime import datetime
	import dns.resolver
	import collections
	import argparse
	import requests
	import time
	import sys
	import os


	gContext = collections.namedtuple("gContext", ["py_ver", "version", "debug", "etag"])
	gContext.py_ver = sys.version_info.major
	gContext.version = "1.0.0"
	gContext.debug = False
	gContext.etag = ""


	def sPrint(message):
	cTime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
	print("%s\t%s" % (cTime, message))
	sys.stdout.flush()
	return


	def createDNSResolver():
	resolver = dns.resolver.Resolver()
	resolver.timeout = 9
	resolver.lifetime = 3
	return resolver


	def queryDNSRecord(dnsResolver, domain, version, retry=10):
	answers = None
	addrGroup = []
	for _ in range(retry):
	if version == 4:
	try:
	if gContext.py_ver > 2:
	answers = dnsResolver.resolve(domain, "A")
	else:
	answers = dnsResolver.query(domain, "A")
	except:
	answers = None
	elif version == 6:
	try:
	if gContext.py_ver > 2:
	answers = dnsResolver.resolve(domain, "AAAA")
	else:
	answers = dnsResolver.query(domain, "AAAA")
	except:
	answers = None
	if answers:
	for addr in answers:
	addrGroup.append(addr.address)
	if addrGroup:
	break
	else:
	time.sleep(1.5)
	if addrGroup:
	addrGroup.sort()
	return (domain, addrGroup[0])
	else:
	return (domain, "")
	return addrGroup


	def fetchPeersHcl():
	try:
	etag = requests.head(PeersURL).headers["Etag"]
	if not gContext.etag:
	gContext.etag = etag
	if gContext.debug:
	sPrint("init data for %s" % (PeersURL,))
	elif gContext.etag == etag:
	if gContext.debug:
	sPrint("no change for %s" % (PeersURL,))
	return []
	except:
	return []
	import hcl

	try:
	if gContext.py_ver > 2:
	peersData = requests.get(PeersURL, timeout=10).content.decode()
	else:
	peersData = requests.get(PeersURL, timeout=10).content
	except Exception as e:
	if gContext.debug:
	sPrint(e)
	return []
	peersGroup = hcl.loads(peersData)
	if not peersGroup:
	return []
	dnsGroup = []
	for peer in peersGroup["peers"]:
	endpoint = peer.get("endpoint", {})
	if endpoint:
	dnsName = (endpoint["address"], endpoint["address_family"])
	if dnsName[0]:
	dnsGroup.append(dnsName)
	dnsGroup.sort(key=lambda x: "%s_%s" % (x[0], x[1]))
	return dnsGroup


	def fetchPeersFake():
	try:
	etag = requests.head(PeersURL).headers["Etag"]
	if not gContext.etag:
	gContext.etag = etag
	if gContext.debug:
	sPrint("init data for %s" % (PeersURL,))
	elif gContext.etag == etag:
	if gContext.debug:
	sPrint("no change for %s" % (PeersURL,))
	return []
	except:
	return []
	try:
	if gContext.py_ver > 2:
	peersData = requests.get(PeersURL, timeout=10).content.decode()
	else:
	peersData = requests.get(PeersURL, timeout=10).content
	except Exception as e:
	if gContext.debug:
	sPrint(e)
	return []
	with open("/etc/higgs/higgs_name.hcl", "w") as fp:
	fp.write(peersData)
	fp.flush()
	dnsGroup = []
	pd = peersData.replace("\r", "").split("\n")
	t = {
	"0": "",
	"1": "",
	}
	st = []
	for i in pd:
	i = i.replace(" ", "")
	if i == "endpoint{":
	st.append(0)
	elif i == "}":
	if st:
	if t["0"]:
	dnsGroup.append((t["0"], t["1"]))
	st.pop()
	if st:
	i = i.split("=")
	if i[0] == "address":
	t["0"] = i[1].replace('"', "")
	elif i[0] == "address_family":
	t["1"] = i[1].replace('"', "")
	dnsGroup.sort(key=lambda x: "%s_%s" % (x[0], x[1]))
	return dnsGroup


	def checkPeers(dnsGroup):
	sp = time.time()
	dnsRecords = {}
	dnsResolver = createDNSResolver()
	for dnsName in dnsGroup:
	if dnsName[1] == "ip4":
	ret = queryDNSRecord(dnsResolver, dnsName[0], 4)
	elif dnsName[1] == "ip6":
	ret = queryDNSRecord(dnsResolver, dnsName[0], 6)
	dnsRecords["%s_%s" % (dnsName[0], dnsName[1])] = ret[1]
	time.sleep(0.05)
	ep = time.time()
	if gContext.debug:
	sPrint("DNS Query Time Spend: %.3fs" % (ep - sp,))
	return dnsRecords


	def updatePeers(dnsRecords):
	with open("/etc/higgs/higgs_name.hcl", "r") as fp:
	pd = fp.readlines()
	t = {
	"0": "",
	"1": "",
	}
	st = []
	for n1, i in enumerate(pd):
	i = i.replace("\r", "").replace("\n", "").replace(" ", "")
	if i == "endpoint{":
	st.append(0)
	elif i == "}":
	if st:
	if t["0"]:
	n2 = n1 - 1
	while True:
	j = pd[n2].replace("\r", "").replace("\n", "").replace(" ", "")
	if j.find("address=") > -1:
	pd[n2] = pd[n2].replace(
	t["0"], dnsRecords["%s_%s" % (t["0"], t["1"])]
	)
	break
	else:
	n2 -= 1
	st.pop()
	if st:
	i = i.split("=")
	if i[0] == "address":
	t["0"] = i[1].replace('"', "")
	elif i[0] == "address_family":
	t["1"] = i[1].replace('"', "")
	with open("/etc/higgs/higgs_addr.hcl", "w") as fp:
	fp.write("".join(pd))
	fp.flush()
	return


	def doSystemAction(systemAction):
	try:
	command = "bash -c '%s'" % (systemAction,)
	if gContext.debug:
	sPrint(command)
	os.system(command)
	except Exception as e:
	sPrint(e)
	return


	def higgsPlusVisor(systemAction, heartBeat, createConfig=False, IsStableNode=False):
	while True:
	# lastDnsRecords=checkPeers(fetchPeersHcl())
	lastDnsRecords = checkPeers(fetchPeersFake())
	if lastDnsRecords:
	break
	else:
	time.sleep(heartBeat)
	shouldRefresh = False
	updatePeers(lastDnsRecords)
	if createConfig:
	sPrint("Finished Create Config At %s" % ("/etc/higgs/higgs_addr.hcl",))
	return
	for n, i in enumerate(lastDnsRecords.items()):
	sPrint("%-3d %s --> %s" % (n + 1, i[0], i[1]))
	sPrint("Finished Initinal Higgs Plus System....")
	while True:
	time.sleep(heartBeat)
	newDnsRecords = checkPeers(fetchPeersFake())
	if len(newDnsRecords) == 0:
	shouldRefresh = False
	elif len(lastDnsRecords) == len(newDnsRecords):
	for name, addr in newDnsRecords.items():
	if lastDnsRecords[name] != addr:
	if (addr) and (not IsStableNode):
	shouldRefresh = True
	sPrint("Detect DNS Record Change %s --> %s" % (name, addr))
	elif len(lastDnsRecords) != len(newDnsRecords):
	shouldRefresh = True
	sPrint("Detect Node Change * <--> *")
	if shouldRefresh:
	shouldRefresh = False
	lastDnsRecords = newDnsRecords
	updatePeers(lastDnsRecords)
	doSystemAction(systemAction)
	sPrint("Finished Refresh Higgs Service !")
	time.sleep(heartBeat * 3)
	return


	def main():
	opt = argparse.ArgumentParser()
	opt.add_argument("-d", "--debug", action="store_true", help="enable debug")
	opt.add_argument(
	"-cc", "--CreateConfig", action="store_true", help="Create Config For Initinal"
	)
	opt.add_argument(
	"-is",
	"--IsStableNode",
	action="store_true",
	help="Is Stable Node (for vps optimize, home/nat needn't, but not suggest enable)",
	)
	opt.add_argument(
	"-sa",
	"--SystemAction",
	type=str,
	default="echo Do Nothing....",
	help="Refresh Command To Use('systemctl reload higgs')",
	)
	opt.add_argument(
	"-hb",
	"--HeartBeat",
	type=int,
	default=30,
	help="Refresh Check Interval(30 seconds)",
	)
	param = opt.parse_args()
	if param.debug:
	gContext.debug = True
	try:
	higgsPlusVisor(
	param.SystemAction, param.HeartBeat, param.CreateConfig, param.IsStableNode
	)
	except Exception as e:
	sPrint("\n%s" % e)
	return


	if __name__ == "__main__":
	main()
	[Unit]
	Description=higgs plus
	After=higgs.service
	Wants=higgs.service

	[Service]
	Restart=always
	ExecStart=/usr/bin/python /usr/sbin/higgs_plus -sa 'systemctl reload higgs'

	[Install]
	WantedBy=multi-user.target