cloud-init problem

certfiles.txt

[loco@td2 ~]$ sudo cat certfiles.txt
[sudo] password for loco:
total 8
4 -rw-r--r--. 1 root root 2240 Oct 21 23:21 stratuscert.pem
4 -rw-r--r--. 1 root root 2496 Oct 21 23:21 blacklakecacert.pem

certs.rb

module UserDataConcern::WriteFiles::Certs
  private
    CERT_PATH = "/etc/pki/ca-trust/source/anchors"
    def ca_cert
      <<~CERT
      - path: #{ CERT_PATH }/blacklakecacert.pem
          content: |
      #{ cacert }
      CERT
    end

    def stratus_cert
      <<~CERT
      - path: #{ CERT_PATH }/stratuscert.pem
          content: |
      #{ stratuscert }
      CERT
    end

    def cacert
      cacert = Rails.application.credentials.certs[:blacklake]
      cacert.each_line.map {|line| "      "+line }.join
    end

    def stratuscert
      cert = Rails.application.credentials.certs[:stratus]
      cert.each_line.map {|line| "      "+line }.join
    end
end

cloud-init.log

[loco@td2 log]$ sudo cat /var/log/cloud-init.log
...
2018-10-21 23:21:35,301 - stages.py[DEBUG]: Running module phone-home (<module 'cloudinit.config.cc_phone_home' from '/usr/lib/python2.7/site-packages/cloudinit/config/cc_phone_home.py'>) with frequency once-per-instance
2018-10-21 23:21:35,302 - handlers.py[DEBUG]: start: modules-final/config-phone-home: running config-phone-home with frequency once-per-instance
2018-10-21 23:21:35,302 - util.py[DEBUG]: Writing to /var/lib/cloud/instances/116010097/sem/config_phone_home - wb: [420] 19 bytes
2018-10-21 23:21:35,303 - util.py[DEBUG]: Restoring selinux mode for /var/lib/cloud/instances/116010097/sem/config_phone_home (recursive=False)
2018-10-21 23:21:35,303 - util.py[DEBUG]: Restoring selinux mode for /var/lib/cloud/instances/116010097/sem/config_phone_home (recursive=False)
2018-10-21 23:21:35,303 - helpers.py[DEBUG]: Running config-phone-home using lock (<FileLock using file '/var/lib/cloud/instances/116010097/sem/config_phone_home'>)
2018-10-21 23:21:35,304 - util.py[DEBUG]: Reading from /etc/ssh/ssh_host_ed25519_key.pub (quiet=False)
2018-10-21 23:21:35,304 - util.py[DEBUG]: Read 82 bytes from /etc/ssh/ssh_host_ed25519_key.pub
2018-10-21 23:21:35,304 - util.py[DEBUG]: Reading from /etc/ssh/ssh_host_rsa_key.pub (quiet=False)
2018-10-21 23:21:35,304 - util.py[DEBUG]: Read 382 bytes from /etc/ssh/ssh_host_rsa_key.pub
2018-10-21 23:21:35,304 - util.py[DEBUG]: Reading from /etc/ssh/ssh_host_ecdsa_key.pub (quiet=False)
2018-10-21 23:21:35,304 - util.py[DEBUG]: Read 162 bytes from /etc/ssh/ssh_host_ecdsa_key.pub
2018-10-21 23:21:35,304 - url_helper.py[DEBUG]: [0/2] open 'https://99.61.131.56/do_hostkeys/fingerprint' with {'url': 'https://99.61.131.56/do_hostkeys/fingerprint', 'headers': {'X-CSRF-Token': 'mUBHYgfASzOxgF5ziitW0jntJDgvN5cxajI+rKi+yA0=', 'Content-Type': 'application/json', 'User-Agent': 'Cloud-Init/0.7.9'}, 'allow_redirects': True, 'method': 'POST', 'timeout': 5.0} configuration
2018-10-21 23:21:35,469 - util.py[WARNING]: Failed to post phone home data to https://99.61.131.56/do_hostkeys/fingerprint in 2 tries
2018-10-21 23:21:35,470 - util.py[DEBUG]: Failed to post phone home data to https://99.61.131.56/do_hostkeys/fingerprint in 2 tries
Traceback (most recent call last):
  File "/usr/lib/python2.7/site-packages/cloudinit/config/cc_phone_home.py", line 170, in handle
    headers=header_dict)
  File "/usr/lib/python2.7/site-packages/cloudinit/util.py", line 888, in read_file_or_url
    exception_cb=exception_cb)
  File "/usr/lib/python2.7/site-packages/cloudinit/url_helper.py", line 269, in readurl
    raise excps[-1]
UrlError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:579)
2018-10-21 23:21:35,476 - handlers.py[DEBUG]: finish: modules-final/config-phone-home: SUCCESS: config-phone-home ran successfully
...

install_certs.rb

module UserDataConcern::RunCmds::InstallCerts
  private
    def install_certs
      <<~INSTALLCERTS
      - update-ca-trust
      INSTALLCERTS
    end
end

phone_home.rb

module UserDataConcern::PhoneHome
  private
    def phone_home
      <<~PHONEHOME
      phone_home:
        url: "https://#{apphost}/do_hostkeys/fingerprint"
        token: #{ServerSession.server_session_token}
        headers:
          X-CSRF-Token: #{ServerSession.csrf_token}
          Content-Type: application/json
        post: [ pub_key_rsa, instance_id ]
        tries: 2
      PHONEHOME
    end

    def apphost
      #Rails.application.credentials.host
      "99.61.131.56"
    end
end

phone_home_patch.rb

module UserDataConcern::WriteFiles::PhoneHomePatch
  private
    def phone_home_patch
      module_dir = '/usr/lib/python2.7/site-packages/cloudinit/config'
      <<~PHPATCH
        - path: #{module_dir}/cc_phone_home.py
            content: |
              # Copyright (C) 2011 Canonical Ltd.
              # Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P.
              #
              # Author: Scott Moser <scott.moser@canonical.com>
              # Author: Juerg Haefliger <juerg.haefliger@hp.com>
              #
              # This file is part of cloud-init. See LICENSE file for license information.

              """
              Phone Home
              ----------
              **Summary:** post data to url

              This module can be used to post data to a remote host after boot is complete.
              If the post url contains the string ``$INSTANCE_ID`` it will be replaced with
              the id of the current instance. Either all data can be posted or a list of
              keys to post. Available keys are:

                  - ``pub_key_rsa``
                  - ``pub_key_ecdsa``
                  - ``pub_key_ed25519``
                  - ``instance_id``
                  - ``hostname``
                  - ``fdqn``

              **Internal name:** ``cc_phone_home``

              **Module frequency:** per instance

              **Supported distros:** all

              **Config keys**::

                  phone_home:
                      url: http://example.com/$INSTANCE_ID/
                      token: abcd1234
                      post:
                          - pub_key_rsa
                          - instance_id
                          - fqdn
                      tries: 10
                      headers: {X-CSRF-Token: 1234567890}
              """

              # Make certain CA certs are installed before running phone_home.
              # This use of subprocess is supposed to be blocking.
              import subprocess

              with open("/home/loco/certfiles.txt", "w") as file:
                subprocess.call(["ls", "-lst", "/etc/pki/ca-trust/source/anchors"], stdout=file)

              subprocess.call(["update-ca-trust"])


              # Now run phone_home
              import json

              from cloudinit import templater
              from cloudinit import util

              from cloudinit.settings import PER_INSTANCE

              frequency = PER_INSTANCE

              POST_LIST_ALL = [
                  'pub_key_rsa',
                  'pub_key_ecdsa',
                  'pub_key_ed25519',
                  'instance_id',
                  'hostname',
                  'fqdn'
              ]


              # phone_home:
              #  url: http://my.foo.bar/$INSTANCE_ID/
              #  post: all
              #  tries: 10
              #
              # phone_home:
              #  url: http://my.foo.bar/$INSTANCE_ID/
              #  token: AbCd09876
              #  post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id, hostname,
              #          fqdn ]
              #  headers: { X-CSRF-Token: 1234567890,
              #             Content-Type: application/json }
              #

              def handle(name, cfg, cloud, log, args):
                  if len(args) != 0:
                      ph_cfg = util.read_conf(args[0])
                  else:
                      if 'phone_home' not in cfg:
                          log.debug(("Skipping module named %s, "
                                     "no 'phone_home' configuration found"), name)
                          return
                      ph_cfg = cfg['phone_home']

                  if 'url' not in ph_cfg:
                      log.warn(("Skipping module named %s, "
                                "no 'url' found in 'phone_home' configuration"), name)
                      return

                  url = ph_cfg['url']
                  token = ph_cfg.get('token', '')
                  post_list = ph_cfg.get('post', 'all')
                  header_dict = ph_cfg.get('headers', {})
                  tries = ph_cfg.get('tries')
                  try:
                      tries = int(tries)
                  except Exception:
                      tries = 10
                      util.logexc(log, "Configuration entry 'tries' is not an integer, "
                                  "using %s instead", tries)

                  if post_list == "all":
                      post_list = POST_LIST_ALL

                  all_keys = {}
                  all_keys['instance_id'] = cloud.get_instance_id()
                  all_keys['hostname'] = cloud.get_hostname()
                  all_keys['fqdn'] = cloud.get_hostname(fqdn=True)

                  pubkeys = {
                      'pub_key_rsa': '/etc/ssh/ssh_host_rsa_key.pub',
                      'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub',
                      'pub_key_ed25519': '/etc/ssh/ssh_host_ed25519_key.pub'
                  }

                  for (name, path) in pubkeys.items():
                      try:
                          all_keys[name] = util.load_file(path)
                      except Exception:
                          util.logexc(log, "%s: failed to open, can not phone home that "
                                      "data!", path)

                  submit_keys = {}
                  for k in post_list:
                      if k in all_keys:
                          submit_keys[k] = all_keys[k]
                      else:
                          submit_keys[k] = None
                          log.warn(("Requested key %s from 'post'"
                                    " configuration list not available"), k)

                  # Get them ready to be posted
                  real_submit_keys = {}
                  for (k, v) in submit_keys.items():
                      if v is None:
                          real_submit_keys[k] = 'N/A'
                      else:
                          real_submit_keys[k] = str(v)

                  # In case the url is parameterized
                  url_params = {
                      'INSTANCE_ID': all_keys['instance_id'],
                  }
                  url = templater.render_string(url, url_params)

                  # Assemble post data payload
                  payload = real_submit_keys.copy()
                  payload.update({ 'token': token })
                  serv_data = json.dumps({ 'serv_data': payload })

                  # Finally
                  try:
                      util.read_file_or_url(url, data=serv_data,
                                            retries=tries-1, sec_between=3,
                                            ssl_details=util.fetch_ssl_details(cloud.paths),
                                            headers=header_dict)
                  except Exception:
                      util.logexc(log, "Failed to post phone home data to %s in %s tries",
                                  url, tries)

              # vi: ts=4 expandtab
      PHPATCH
    end
end

run_cmds.rb

module UserDataConcern::RunCmds
  include DisableETH1
  include InstallCerts
  private
    def run_cmds
      <<~RUNCMDS
      runcmd:
        #{ disable_eth1 }
        #{ install_certs }
      RUNCMDS
    end
end

user_data_concern.rb

module UserDataConcern
  extend ActiveSupport::Concern
  include WriteFiles
  include RunCmds
  include Users
  include SetUserPasswords
  include PhoneHome

  private
    def user_data
      <<~USERDATA
        #cloud-config

        #{ write_files }
        #{ run_cmds}
        #{ users }
        #{ set_user_pws }
        #{ phone_home }
      USERDATA
    end
end

write_files.rb

module UserDataConcern::WriteFiles
  include Certs
  include PhoneHomePatch
  include SSHDConfig

  private
    def write_files
      <<~WRITE
      write_files:
        #{ ca_cert }
        #{ stratus_cert }
        #{ phone_home_patch }
      WRITE
    end
end