ACME EVH integration

acme-evh.py

'''
Modified https://github.com/diafygi/acme-tiny/blob/master/acme_tiny.py for Avi Controller
'''

'''
Parameters -
    user            - Avi user name
    password        - Password of the above user
    tenant          - Avi tenant name
    dryrun          - True/False. If True letsencrypt's staging server will be used.
    virtualservice  - virtualservice to update

Useful links -
    Ratelimiting - https://letsencrypt.org/docs/rate-limits/
'''

import os, subprocess, json, base64, binascii, time, hashlib, re
from urllib.request import urlopen, Request # Python 3
from tempfile import NamedTemporaryFile

from avi.sdk.avi_api import ApiSession

DEFAULT_CA = "https://acme-v02.api.letsencrypt.org" # DEPRECATED! USE DEFAULT_DIRECTORY_URL INSTEAD
DEFAULT_DIRECTORY_URL = "https://acme-v02.api.letsencrypt.org/directory"
DEFAULT_STAGING_DIRECTORY_URL = "https://acme-staging-v02.api.letsencrypt.org/directory"
ACCOUNT_KEY_PATH = "/var/lib/avi/ca/private/letsencrypt.key"

def get_crt(user, password, tenant, api_version, virtualservice, csr, CA=DEFAULT_CA, disable_check=False, directory_url=DEFAULT_DIRECTORY_URL, contact=None ):
    directory, acct_headers, alg, jwk = None, None, None, None # global variables

    # helper functions - base64 encode for jose spec
    def _b64(b):
        return base64.urlsafe_b64encode(b).decode('utf8').replace("=", "")

    # helper function - run external commands
    def _cmd(cmd_list, stdin=None, cmd_input=None, err_msg="Command Line Error"):
        proc = subprocess.Popen(cmd_list, stdin=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        out, err = proc.communicate(cmd_input)
        if proc.returncode != 0:
            raise IOError("{0}\n{1}".format(err_msg, err))
        return out

    # helper function - make request and automatically parse json response
    def _do_request(url, data=None, err_msg="Error", depth=0):
        try:
            resp = urlopen(Request(url, data=data, headers={"Content-Type": "application/jose+json", "User-Agent": "acme-tiny"}))
            resp_data, code, headers = resp.read().decode("utf8"), resp.getcode(), resp.headers
        except IOError as e:
            resp_data = e.read().decode("utf8") if hasattr(e, "read") else str(e)
            code, headers = getattr(e, "code", None), {}
        try:
            resp_data = json.loads(resp_data) # try to parse json results
        except ValueError:
            pass # ignore json parsing errors
        if depth < 100 and code == 400 and resp_data['type'] == "urn:ietf:params:acme:error:badNonce":
            raise IndexError(resp_data) # allow 100 retrys for bad nonces
        if code not in [200, 201, 204]:
            raise ValueError("{0}:\nUrl: {1}\nData: {2}\nResponse Code: {3}\nResponse: {4}".format(err_msg, url, data, code, resp_data))
        return resp_data, code, headers

    # helper function - make signed requests
    def _send_signed_request(url, payload, err_msg, depth=0):
        payload64 = "" if payload is None else _b64(json.dumps(payload).encode('utf8'))
        new_nonce = _do_request(directory['newNonce'])[2]['Replay-Nonce']
        protected = {"url": url, "alg": alg, "nonce": new_nonce}
        protected.update({"jwk": jwk} if acct_headers is None else {"kid": acct_headers['Location']})
        protected64 = _b64(json.dumps(protected).encode('utf8'))
        protected_input = "{0}.{1}".format(protected64, payload64).encode('utf8')
        out = _cmd(["openssl", "dgst", "-sha256", "-sign", ACCOUNT_KEY_PATH], stdin=subprocess.PIPE, cmd_input=protected_input, err_msg="OpenSSL Error")
        data = json.dumps({"protected": protected64, "payload": payload64, "signature": _b64(out)})
        try:
            return _do_request(url, data=data.encode('utf8'), err_msg=err_msg, depth=depth)
        except IndexError: # retry bad nonces (they raise IndexError)
            return _send_signed_request(url, payload, err_msg, depth=(depth + 1))

    # helper function - poll until complete
    def _poll_until_not(url, pending_statuses, err_msg):
        result, t0 = None, time.time()
        while result is None or result['status'] in pending_statuses:
            assert (time.time() - t0 < 3600), "Polling timeout" # 1 hour timeout
            time.sleep(0 if result is None else 2)
            result, _, _ = _send_signed_request(url, None, err_msg)
        return result

    session = ApiSession('localhost', user, password, tenant=tenant, api_version=api_version)

    def _do_request_avi(url, method, data=None, error_msg="Error"):
        rsp = None
        if method == "GET":
            rsp = session.get(url)
        elif method == "POST":
            rsp = session.post(url, data=data)
        elif method == "PATCH":
            rsp = session.patch(url, data)
        elif method == "PUT":
            rsp = session.put(url, data=data)
        elif method == "DELETE":
            rsp = session.delete(url)
        else:
            raise Exception("Unsupported API method")
        if rsp.status_code >= 300:
            err = error_msg + " url - {}. Method - {}. Response status - {}. Response - ".format(url,method,rsp.status_code,rsp.json())
            raise Exception(err)
        return rsp

    if os.path.exists(ACCOUNT_KEY_PATH):
        print ("Reusing account key.")
    else:
        print ("Account key not found. Generating account key...")
        out = _cmd(["openssl", "genrsa", "4096"], err_msg="OpenSSL Error")
        with open(ACCOUNT_KEY_PATH, 'w') as f:
            f.write(out.decode("utf-8"))

    # parse account key to get public key
    print ("Parsing account key...")
    out = _cmd(["openssl", "rsa", "-in", ACCOUNT_KEY_PATH, "-noout", "-text"], err_msg="OpenSSL Error")
    pub_pattern = r"modulus:[\s]+?00:([a-f0-9\:\s]+?)\npublicExponent: ([0-9]+)"
    pub_hex, pub_exp = re.search(pub_pattern, out.decode('utf8'), re.MULTILINE|re.DOTALL).groups()
    pub_exp = "{0:x}".format(int(pub_exp))
    pub_exp = "0{0}".format(pub_exp) if len(pub_exp) % 2 else pub_exp
    alg = "RS256"
    jwk = {
        "e": _b64(binascii.unhexlify(pub_exp.encode("utf-8"))),
        "kty": "RSA",
        "n": _b64(binascii.unhexlify(re.sub(r"(\s|:)", "", pub_hex).encode("utf-8"))),
    }
    accountkey_json = json.dumps(jwk, sort_keys=True, separators=(',', ':'))
    thumbprint = _b64(hashlib.sha256(accountkey_json.encode('utf8')).digest())

    # find domains
    print ("Parsing CSR...")
    out = _cmd(["openssl", "req", "-in", csr, "-noout", "-text"], err_msg="Error loading {0}".format(csr))
    domains = set([])
    common_name = re.search(r"Subject:.*? CN\s?=\s?([^\s,;/]+)", out.decode('utf8'))
    if common_name is not None:
        domains.add(common_name.group(1))
    subject_alt_names = re.search(r"X509v3 Subject Alternative Name: (?:critical)?\n +([^\n]+)\n", out.decode('utf8'), re.MULTILINE|re.DOTALL)
    if subject_alt_names is not None:
        for san in subject_alt_names.group(1).split(", "):
            if san.startswith("DNS:"):
                domains.add(san[4:])
    print ("Found domains: {0}".format(", ".join(domains)))

    # get the ACME directory of urls
    print ("Getting directory...")
    directory_url = CA + "/directory" if CA != DEFAULT_CA else directory_url # backwards compatibility with deprecated CA kwarg
    directory, _, _ = _do_request(directory_url, err_msg="Error getting directory")
    print ("Directory found!")

    # create account, update contact details (if any), and set the global key identifier
    print ("Registering account...")
    reg_payload = {"termsOfServiceAgreed": True}
    account, code, acct_headers = _send_signed_request(directory['newAccount'], reg_payload, "Error registering")
    print ("Registered!" if code == 201 else "Already registered!")
    if contact is not None:
        account, _, _ = _send_signed_request(acct_headers['Location'], {"contact": contact}, "Error updating contact details")
        print ("Updated contact details:\n{0}".format("\n".join(account['contact'])))

    # create a new order
    print ("Creating new order...")
    order_payload = {"identifiers": [{"type": "dns", "value": d} for d in domains]}
    order, _, order_headers = _send_signed_request(directory['newOrder'], order_payload, "Error creating new order")
    print ("Order created!")

    # get the authorizations that need to be completed
    for auth_url in order['authorizations']:
        authorization, _, _ = _send_signed_request(auth_url, None, "Error getting challenges")
        domain = authorization['identifier']['value']
        print ("Verifying {0}...".format(domain))

        # find the http-01 challenge and write the challenge file
        challenge = [c for c in authorization['challenges'] if c['type'] == "http-01"][0]
        token = re.sub(r"[^A-Za-z0-9_\-]", "_", challenge['token'])
        keyauthorization = "{0}.{1}".format(token, thumbprint)

        # Update vs

#        rsp = _do_request_avi("virtualservice?name=evh-parent".format(virtualservice), "GET").json()
#        rsp = _do_request_avi("virtualservice?name="+.format(virtualservice), "GET").json()
        rsp = _do_request_avi("virtualservice?name="+virtualservice, "GET").json()

        if rsp['count'] == 0:
            raise Exception("Could not find a VS with common name = {}".format(virtualservice))

        vs_uuid = rsp["results"][0]["uuid"]
        print ("Found vs {} with fqdn {}".format(vs_uuid, domain))
        # Check if the vs is servering on port 80
        serving_on_port_80 = False
        service_on_port_80_data = None
        for service in rsp["results"][0]["services"]:
            if service["port"] == 80:
                serving_on_port_80 = True
                print ("VS serving on port 80")
                break

        # create HTTP policy
        httppolicy_data = {
            "name": (domain + "LetsEncryptHTTPpolicy"),
            "http_security_policy": {
            "rules": [{
                "name": "Rule 1",
                "index": 1,
                "enable": True,
                "match": {
                    "vs_port": {
                        "match_criteria": "IS_IN",
                        "ports": [80]
                    },
                    "path": {
                        "match_criteria": "CONTAINS",
                        "match_case": "SENSITIVE",
                        "match_str": [
                        ".well-known/acme-challenge/{}".format(token)
                        ]
                    }
                },
                "action": {
                    "action": "HTTP_SECURITY_ACTION_SEND_RESPONSE",
                    "status_code": "HTTP_LOCAL_RESPONSE_STATUS_CODE_200",
                    "file": {
                        "content_type": "text/plain",
                        "file_content": keyauthorization
                    }
                }
            }]
            },
            "is_internal_policy": False
        }
        try:
            rsp = _do_request_avi("httppolicyset", "POST", data=httppolicy_data).json()
            httppolicy_uuid = rsp["uuid"]
            print ("Created HTTP policy with uuid {}".format(httppolicy_uuid))

            patch_data = {"add" : {"http_policies": [{"http_policy_set_ref": "/api/httppolicyset/{}".format(httppolicy_uuid), "index":1000001}]}}
            if not serving_on_port_80:
                # Add to port to virtualservice
                print ("Adding port 80 to VS")
                service_on_port_80_data = {          
                    "enable_http2": False,
                    "enable_ssl": False,
                    "port": 80,
                    "port_range_end": 80
                }
                patch_data["add"]["services"] = [service_on_port_80_data]
            _do_request_avi("virtualservice/{}".format(vs_uuid), "PATCH", patch_data)
            print ("Added HTTPPolicy to VS")

            # check that the file is in place
            try:
                wellknown_url = "http://{0}/.well-known/acme-challenge/{1}".format(domain, token)
                assert (disable_check or _do_request(wellknown_url)[0] == keyauthorization)
            except (AssertionError, ValueError) as e:
                raise ValueError("Wrote file to {0}, but couldn't download {1}: {2}".format('wellknown_path', 'wellknown_url', e))

            print ("Challenge Completed, notifying LetsEncrypt")
            # say the challenge is done
            _send_signed_request(challenge['url'], {}, "Error submitting challenges: {0}".format(domain))
            authorization = _poll_until_not(auth_url, ["pending"], "Error checking challenge status for {0}".format(domain))
            if authorization['status'] != "valid":
                raise ValueError("Challenge did not pass for {0}: {1}".format(domain, authorization))
            print ("Challenge Passed")

        finally:
            # Update the vs
            patch_data = {"delete" : {"http_policies": [{"http_policy_set_ref": "/api/httppolicyset/{}".format(httppolicy_uuid), "index":1000001}]}}
            if not serving_on_port_80:
                patch_data["delete"]["services"] = [service_on_port_80_data]
            _do_request_avi("virtualservice/{}".format(vs_uuid), "PATCH", patch_data)
            print ("Removed HTTPPolicy from VS")
            _do_request_avi("httppolicyset/{}".format(httppolicy_uuid), "DELETE")
            print ("Deleted HTTPPolicy")

        print ("{0} verified!".format(domain))

    # finalize the order with the csr
    print ("Signing certificate...")
    csr_der = _cmd(["openssl", "req", "-in", csr, "-outform", "DER"], err_msg="DER Export Error")
    _send_signed_request(order['finalize'], {"csr": _b64(csr_der)}, "Error finalizing order")

    # poll the order to monitor when it's done
    order = _poll_until_not(order_headers['Location'], ["pending", "processing"], "Error checking order status")
    if order['status'] != "valid":
        raise ValueError("Order failed: {0}".format(order))

    # download the certificate
    certificate_pem, _, _ = _send_signed_request(order['certificate'], None, "Certificate download failed")
    print ("Certificate signed!")

    return certificate_pem

def certificate_request(csr, common_name, kwargs):
    user = kwargs.get('user', None)
    password = kwargs.get('password', None)
    tenant = kwargs.get('tenant', '*')
    dry_run = kwargs.get('dryrun', '')
    contact = kwargs.get('contact', None)
    api_version = kwargs.get('api_version', '20.1.1')
    virtualservice = kwargs.get('virtualservice')

    if dry_run.lower() == "true":
        dry_run = True
    else:
        dry_run = False

    directory_url = DEFAULT_DIRECTORY_URL
    if dry_run:
        directory_url = DEFAULT_STAGING_DIRECTORY_URL

    csr_temp_file = NamedTemporaryFile(mode='w',delete=False)
    csr_temp_file.close()

    with open(csr_temp_file.name, 'w') as f:
        f.write(csr)

    signed_crt = None
    try:
        signed_crt = get_crt(user, password, tenant, api_version, virtualservice, csr_temp_file.name, directory_url=directory_url, contact=contact)
    finally:
        os.remove(csr_temp_file.name)

    print (signed_crt)
    return signed_crt

Hi, thanks very much for providing this! A couple of notes / questions:

1. Of note, this was failing for me until I put an entry for each parameter - I thought this may help others as it was not 100% clear I need an entry for virtualservice which was not included in the SNI approach
2. I'm assuming as this is for a lab there is currently no renewal built in. Any plans to?

There are no parameters correct. There actually should be a renewal as the tinyacme also renews, and avi should automatic call the script when a certificate is due to expire. Get Outlook for iOS<https://aka.ms/o0ukef>

…

________________________________ From: mackov83 ***@***.***> Sent: Monday, August 30, 2021 2:53:13 PM To: srobroek ***@***.***> Cc: Sjors Robroek ***@***.***>; Author ***@***.***> Subject: Re: srobroek/acme-evh.py @mackov83 commented on this gist.

________________________________ Hi, thanks very much for providing this! A couple of notes / questions: 1. Of note, this was failing for me until I put an entry for each parameter 2. I'm assuming as this is for a lab there is current no renewal built in. Any plans to? — You are receiving this because you authored the thread. Reply to this email directly, view it on GitHub<https://gist.github.com/d71cbe93813bc45f4c580cc4d6d8090f#gistcomment-3876767>, or unsubscribe<https://github.com/notifications/unsubscribe-auth/ABWLXZELXEJWYKY2BBZPO5DT7N5LTANCNFSM5DBZ5F4Q>. Triage notifications on the go with GitHub Mobile for iOS<https://apps.apple.com/app/apple-store/id1477376905?ct=notification-email&mt=8&pt=524675> or Android<https://play.google.com/store/apps/details?id=com.github.android&referrer=utm_campaign%3Dnotification-email%26utm_medium%3Demail%26utm_source%3Dgithub>.

No default parameters, apologies Get Outlook for iOS<https://aka.ms/o0ukef>

…

________________________________ From: Sjors Robroek ***@***.***> Sent: Monday, August 30, 2021 6:01:53 PM To: srobroek ***@***.***>; srobroek ***@***.***> Cc: Author ***@***.***> Subject: Re: srobroek/acme-evh.py There are no parameters correct. There actually should be a renewal as the tinyacme also renews, and avi should automatic call the script when a certificate is due to expire. Get Outlook for iOS<https://aka.ms/o0ukef>

________________________________ From: mackov83 ***@***.***> Sent: Monday, August 30, 2021 2:53:13 PM To: srobroek ***@***.***> Cc: Sjors Robroek ***@***.***>; Author ***@***.***> Subject: Re: srobroek/acme-evh.py @mackov83 commented on this gist.

________________________________ Hi, thanks very much for providing this! A couple of notes / questions: 1. Of note, this was failing for me until I put an entry for each parameter 2. I'm assuming as this is for a lab there is current no renewal built in. Any plans to? — You are receiving this because you authored the thread. Reply to this email directly, view it on GitHub<https://gist.github.com/d71cbe93813bc45f4c580cc4d6d8090f#gistcomment-3876767>, or unsubscribe<https://github.com/notifications/unsubscribe-auth/ABWLXZELXEJWYKY2BBZPO5DT7N5LTANCNFSM5DBZ5F4Q>. Triage notifications on the go with GitHub Mobile for iOS<https://apps.apple.com/app/apple-store/id1477376905?ct=notification-email&mt=8&pt=524675> or Android<https://play.google.com/store/apps/details?id=com.github.android&referrer=utm_campaign%3Dnotification-email%26utm_medium%3Demail%26utm_source%3Dgithub>.

Hi, this all seemed to work the first time around when I generated with staging cert. I then deleted the staging cert and reattempted and since then it has not worked. Any chance you might be able to tell from this output what's going wrong?

[2021-09-05 15:21:18,023] ERROR [error_handler.process_exception:102] Error in POST /api/sslkeyandcertificate :
Traceback (most recent call last):
  File "/tmp/YADVQNGC.py", line 250, in get_crt
AssertionError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/avi/python/lib/avi/util/ssl_utils.py", line 755, in fetch_cert_from_script
    certificate = script.certificate_request(csr, common_name, script_args)
  File "/tmp/YADVQNGC.py", line 316, in certificate_request
  File "/tmp/YADVQNGC.py", line 252, in get_crt
ValueError: Wrote file to wellknown_path, but couldn't download wellknown_url:

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.8/dist-packages/django/core/handlers/base.py", line 132, in get_response
    response = wrapped_callback(request, *callback_args, **callback_kwargs)
  File "/usr/local/lib/python3.8/dist-packages/django/views/generic/base.py", line 71, in view
    return self.dispatch(request, *args, **kwargs)
  File "/usr/local/lib/python3.8/dist-packages/django/views/decorators/csrf.py", line 58, in wrapped_view
    return view_func(*args, **kwargs)
  File "/usr/local/lib/python3.8/dist-packages/rest_framework/views.py", line 404, in dispatch
    response = self.handle_exception(exc)
  File "/usr/local/lib/python3.8/dist-packages/rest_framework/views.py", line 401, in dispatch
    response = handler(request, *args, **kwargs)
  File "/opt/avi/python/lib/avi/rest/views.py", line 1057, in post
    rsp = self.do_post_transaction(request, *args, **kwargs)
  File "/opt/avi/python/lib/avi/infrastructure/db_transaction.py", line 51, in inner
    ret = f(*args, **kwargs)
  File "/opt/avi/python/lib/avi/rest/views.py", line 1054, in do_post_transaction
    return self.do_post_list(request, *args, **kwargs)
  File "/opt/avi/python/lib/avi/rest/views.py", line 1050, in do_post_list
    return self.do_post_action(request, *args, **kwargs)
  File "/opt/avi/python/lib/avi/rest/api_version.py", line 284, in _apply_versioning
    rsp = f(*args, **kwargs)
  File "/opt/avi/python/lib/avi/rest/views.py", line 1044, in do_post_action
    return self.create(request, *args, **kwargs)
  File "/opt/avi/python/lib/avi/rest/api_perf.py", line 116, in _perf
    rsp = f(*args, **kwargs)
  File "/opt/avi/python/lib/avi/rest/mixins.py", line 787, in create
    new_pb = self.pre_save_transform(model, new_pb, request)
  File "/opt/avi/python/lib/avi/rest/api_perf.py", line 116, in _perf
    rsp = f(*args, **kwargs)
  File "/opt/avi/python/lib/avi/rest/mixins.py", line 656, in pre_save_transform
    pb_message_pre_save_transform(model, new_pb, tenant_name=tenant_name,
  File "/opt/avi/python/lib/avi/util/pb_routines.py", line 115, in pb_message_pre_save_transform
    pb_resolve.run_custom_resolve(message, tenant_name=tenant_name, old_pb=old_pb)
  File "/opt/avi/python/lib/avi/util/pb_resolve.py", line 102, in run_custom_resolve
    func(message, old_pb=old_pb)
  File "/opt/avi/python/lib/avi/util/pb_resolve.py", line 1683, in SSLKeyAndCertificateResolve
    extend_req_cert(message)
  File "/opt/avi/python/lib/avi/util/ssl_utils.py", line 638, in extend_req_cert
    cert, _, _ = fetch_cert_from_script(cert_pb.certificate_signing_request,
  File "/opt/avi/python/lib/avi/util/ssl_utils.py", line 762, in fetch_cert_from_script
    raise ServerException("Error from certificate management service: %s.\n STDOUT - '%s'.\n STDERR - '%s'." % (str(e), out , err))
avi.rest.error_list.ServerException: Error from certificate management service: Wrote file to wellknown_path, but couldn't download wellknown_url: .
 STDOUT - 'Reusing account key.
Parsing account key...
Parsing CSR...
Found domains: sub.domain.com
Getting directory...
Directory found!
Registering account...
Already registered!
Creating new order...
Order created!
Verifying sub.domain.com...
Found vs virtualservice-c8d6bc7c-df60-4ab4-8dc5-9b873c802234 with fqdn sub.domain.com
VS serving on port 80
Created HTTP policy with uuid httppolicyset-cde85f61-5f79-46e2-8672-c9d7b9b813a9
Added HTTPPolicy to VS
Removed HTTPPolicy from VS
Deleted HTTPPolicy
'.
 STDERR - ''.

Hi, this all seemed to work the first time around when I generated with staging cert. I then deleted the staging cert and reattempted and since then it has not worked. Any chance you might be able to tell from this output what's going wrong?

[2021-09-05 15:21:18,023] ERROR [error_handler.process_exception:102] Error in POST /api/sslkeyandcertificate :
Traceback (most recent call last):
  File "/tmp/YADVQNGC.py", line 250, in get_crt
AssertionError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/opt/avi/python/lib/avi/util/ssl_utils.py", line 755, in fetch_cert_from_script
    certificate = script.certificate_request(csr, common_name, script_args)
  File "/tmp/YADVQNGC.py", line 316, in certificate_request
  File "/tmp/YADVQNGC.py", line 252, in get_crt
ValueError: Wrote file to wellknown_path, but couldn't download wellknown_url:

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.8/dist-packages/django/core/handlers/base.py", line 132, in get_response
    response = wrapped_callback(request, *callback_args, **callback_kwargs)
  File "/usr/local/lib/python3.8/dist-packages/django/views/generic/base.py", line 71, in view
    return self.dispatch(request, *args, **kwargs)
  File "/usr/local/lib/python3.8/dist-packages/django/views/decorators/csrf.py", line 58, in wrapped_view
    return view_func(*args, **kwargs)
  File "/usr/local/lib/python3.8/dist-packages/rest_framework/views.py", line 404, in dispatch
    response = self.handle_exception(exc)
  File "/usr/local/lib/python3.8/dist-packages/rest_framework/views.py", line 401, in dispatch
    response = handler(request, *args, **kwargs)
  File "/opt/avi/python/lib/avi/rest/views.py", line 1057, in post
    rsp = self.do_post_transaction(request, *args, **kwargs)
  File "/opt/avi/python/lib/avi/infrastructure/db_transaction.py", line 51, in inner
    ret = f(*args, **kwargs)
  File "/opt/avi/python/lib/avi/rest/views.py", line 1054, in do_post_transaction
    return self.do_post_list(request, *args, **kwargs)
  File "/opt/avi/python/lib/avi/rest/views.py", line 1050, in do_post_list
    return self.do_post_action(request, *args, **kwargs)
  File "/opt/avi/python/lib/avi/rest/api_version.py", line 284, in _apply_versioning
    rsp = f(*args, **kwargs)
  File "/opt/avi/python/lib/avi/rest/views.py", line 1044, in do_post_action
    return self.create(request, *args, **kwargs)
  File "/opt/avi/python/lib/avi/rest/api_perf.py", line 116, in _perf
    rsp = f(*args, **kwargs)
  File "/opt/avi/python/lib/avi/rest/mixins.py", line 787, in create
    new_pb = self.pre_save_transform(model, new_pb, request)
  File "/opt/avi/python/lib/avi/rest/api_perf.py", line 116, in _perf
    rsp = f(*args, **kwargs)
  File "/opt/avi/python/lib/avi/rest/mixins.py", line 656, in pre_save_transform
    pb_message_pre_save_transform(model, new_pb, tenant_name=tenant_name,
  File "/opt/avi/python/lib/avi/util/pb_routines.py", line 115, in pb_message_pre_save_transform
    pb_resolve.run_custom_resolve(message, tenant_name=tenant_name, old_pb=old_pb)
  File "/opt/avi/python/lib/avi/util/pb_resolve.py", line 102, in run_custom_resolve
    func(message, old_pb=old_pb)
  File "/opt/avi/python/lib/avi/util/pb_resolve.py", line 1683, in SSLKeyAndCertificateResolve
    extend_req_cert(message)
  File "/opt/avi/python/lib/avi/util/ssl_utils.py", line 638, in extend_req_cert
    cert, _, _ = fetch_cert_from_script(cert_pb.certificate_signing_request,
  File "/opt/avi/python/lib/avi/util/ssl_utils.py", line 762, in fetch_cert_from_script
    raise ServerException("Error from certificate management service: %s.\n STDOUT - '%s'.\n STDERR - '%s'." % (str(e), out , err))
avi.rest.error_list.ServerException: Error from certificate management service: Wrote file to wellknown_path, but couldn't download wellknown_url: .
 STDOUT - 'Reusing account key.
Parsing account key...
Parsing CSR...
Found domains: sub.domain.com
Getting directory...
Directory found!
Registering account...
Already registered!
Creating new order...
Order created!
Verifying sub.domain.com...
Found vs virtualservice-c8d6bc7c-df60-4ab4-8dc5-9b873c802234 with fqdn sub.domain.com
VS serving on port 80
Created HTTP policy with uuid httppolicyset-cde85f61-5f79-46e2-8672-c9d7b9b813a9
Added HTTPPolicy to VS
Removed HTTPPolicy from VS
Deleted HTTPPolicy
'.
 STDERR - ''.

Strange, i haven't been able to reproduce this in my own lab. It could be that there's some dangling HTTP policy left?

I did some further testing and the only way I could get this to work was:

1. Remove all certificates attached to the https evh
2. Run the certificate generation process - this time it worked without issue
3. Attach the certificates back to the https evh

As I had multiple virtual hosts I had to perform this process more than once, but while not ideal (sites would be down with certs missing) it worked. It will be interesting to see what happens when renewals are due...

I did some further testing and the only way I could get this to work was:

1. Remove all certificates attached to the https evh

2. Run the certificate generation process - this time it worked without issue

3. Attach the certificates back to the https evh

As I had multiple virtual hosts I had to perform this process more than once, but while not ideal (sites would be down with certs missing) it worked. It will be interesting to see what happens when renewals are due...

Just to check, do you have SSL anywhere redirects configured by any chance? That's what would be breaking it for you since once you've got the hostname up in your ssl profile AVI would now be redirecting the traffic there and as such sending a 302 redirecting to https which would fail.

What you need is a separate vs responding on port 80 and then, if you want a rewrite to https you'd need to create a HTTP response rule such as this:

group letsencrypt in my case is just a string group matching /.well-known/acme-challenge.

That way traffic directed towards the acme challenge will not be rewritten.

Hi, this seemed to work fine for a while but I have noticed that it has stopped working again. I tried deleting the cert completely for a test domain and am getting the following error:

Error from certificate management service: local variable 'httppolicy_uuid' referenced before assignment. STDOUT - 'Reusing account key. Parsing account key... Parsing CSR... Found domains: sub.domain.com Getting directory... Directory found! Registering account... Already registered! Creating new order... Order created! Verifying sub.domain.com... Found vs virtualservice-950baec1-8b35-41a3-8bd8-5316fce08939 with fqdn sub.domain.com VS serving on port 80 '. STDERR - ''.

I have upgraded AVI / NSXALB a few times, so I was wondering if the code has changed. My version is currently 21.1.2-2p2.

Also, is it normal for these SSL renewals to run every minute? The controller alerts are filled with attempt after attempt that run every minute

error: Encountered an error on POST request to URL https://localhost//api/sslkeyandcertificate/sslkeyandcertificate-685f5d92-3c67-4558-b109-cec37ce762ae/renew: HTTP code: 400; error from Avi: map[error:Error from certificate management service: local variable 'httppolicy_uuid' referenced before assignment. STDOUT - 'Decrypting sensitive fields for Certificate management profile '. STDERR - ''. traceback:Traceback (most recent call last): File "/tmp/NAXJXYHG.py", line 229, in get_crt File "/tmp/NAXJXYHG.py", line 101, in _do_request_avi File "/opt/avi/python/lib/avi/sdk/avi_api.py", line 122, in json raise APIError('HTTP Error: %d Error Msg %s' % ( avi.sdk.avi_api.APIError: ('HTTP Error: 409 Error Msg {"error": "Http policy set with this Name and Tenant ref already exists."}', <Response [409]>) During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/opt/avi/python/lib/avi/util/ssl_utils.py", line 755, in fetch_cert_from_script certificate = script.certificate_request(csr, common_name, script_args) File "/tmp/NAXJXYHG.py", line 316, in certificate_request File "/tmp/NAXJXYHG.py", line 264, in get_crt UnboundLocalError: local variable 'httppolicy_uuid' referenced before assignment During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/opt/avi/python/bin/portal/api/views_ssl_custom.py", line 183, in post_transaction ssl_pb, cmp_stdout, cmp_stderr = renew_management_profile_ssl_certificate(ssl_pb, certificate_management_profile) File "/opt/avi/python/lib/avi/util/ssl_utils.py", line 2076, in renew_management_profile_ssl_certificate cert, out, err = fetch_cert_from_script(cert_pb.certificate_signing_request, File "/opt/avi/python/lib/avi/util/ssl_utils.py", line 762, in fetch_cert_from_script raise ServerException("Error from certificate management service: %s.\n STDOUT - '%s'.\n STDERR - '%s'." % (str(e), out , err)) avi.rest.error_list.ServerException: Error from certificate management service: local variable 'httppolicy_uuid' referenced before assignment. STDOUT - 'Decrypting sensitive fields for Certificate management profile '. STDERR - ''. ]

Hmmm, I think it could have been a misconfiguration that crept into my HTTP evh. After rolling back to 21.1.1 I started getting other errors, but I noticed that during the rollback process that other services went missing - I read somewhere it is like a snapshot rollback?

I deleted the HTTP evh and recreated it with only necessary settings and it worked fine after that. My redirect that was previously in place was exactly the same as yours, but I did notice that I had some other http security configs in place and I am not sure why