_ [dc2] (docker-compose)+(ytt).md

This is a working space for a ytt based project.

It intends to build composefiles for a wordpress/docker project I am working on.

Live playground: https://get-ytt.io/#gist:https://gist.github.com/JollyWizard/6501758afc6162508ab7c3ddb84c1fbe

Gist (Sources): https://gist.github.com/JollyWizard/6501758afc6162508ab7c3ddb84c1fbe

compose.star

load("@ytt:struct","struct")

load("@ytt:data", "data")

composefile_version = "3.2"

def _composefile_version(context):
  """
    Property Accessor for Composefile format version.

    @TODO: make this optional config in `@ytt/data`
  """
  return composefile_version
end

def _services(context, *service_names):
  """
    Return a set of service definitions, to be anchored to a `services:` key.
  """
  services = {}

  for service_name in service_names:
    services[service_name] = _service(context, service_name)
  end

  return services
end

def _service(context, service_name):
  """
    Return a single service definition, to be anchored to a `{service-name}:` key.
    
    This should include the runtime configuration, and the debug details.
    Make debug an optional param later.
  """

  def _profile_chain(profile):
    """
      Get a list of profiles that includes all ancestors.
      
      This list will be navigated to provide all hierarchy info.
      
      @TODO Move to public utility function.
    """
    
    if SERVICE_PROFILE.EXTEND_PATH not in profile:
      return [profile]
    end
    
    extend_path = profile[SERVICE_PROFILE.EXTEND_PATH]
    
    #! find ancestors first
    #! TODO: Does this need to be reversed?
    r = list_finddicts_byname(context[SERVICE_PROFILES], *extend_path)
    #! add current
    r.append(profile)
    return r
  end

  def resolve_environment(_profile):
    """ 
      Resolve the service profile into a list of environment key-values definitions. 
    """
    r = []

    for profile in _profile_chain(_profile):
      if SERVICE_PROFILE.ENVIRONMENT_GROUPS in profile:
        groups = profile[SERVICE_PROFILE.ENVIRONMENT_GROUPS]
        toAdd = [definition for definition in environmentgroup_get_definitions(context, *groups)]
        for add in toAdd: r.append(add)
      end
    end

    return r
  end

  def resolve_volumes(profile):
    """
      Resolve the service profile into a list of volume definitions.
      * copy in literals from profile.properties.
    """
    r = []

    for profile in _profile_chain(profile):

      if SERVICE_PROFILE.PROPERTIES in profile: 
        properties = profile[SERVICE_PROFILE.PROPERTIES]
        if 'volumes' in properties:
          for definition in properties['volumes']:
            r.append(definition)
          end
        end
      end

      if SERVICE_PROFILE.VOLUME_GROUPS in profile:
        for volumegroup in profile[SERVICE_PROFILE.VOLUME_GROUPS]:
          volumedetails = volumegroup_get_volumedetails(context, volumegroup)

          toAdd = [definition for definition in volumedetails_get_definitions(context, *volumedetails)]
          for add in toAdd: r.append(add)
        end
      end
    end
    
    return r
  end

  # Build service profile.

  profile = _profile(context, service_name)
  if profile == None:
    return {'does-not-exist': service_name}
  end
  if SERVICE_PROFILE.PROPERTIES in profile:
    r = {k:v for k,v in profile[SERVICE_PROFILE.PROPERTIES].items()}
  end
  r['environment'] = resolve_environment(profile)
  r['volumes'] = resolve_volumes(profile)
  return r
end

def _profile(context, service_name):
  """
    Return the profile for the service name.
  """
  profiles = context[SERVICE_PROFILES]
  profile = list_finddict_byname(profiles, service_name)
  return profile
end 

def _data_load():
  """
    Load the underlying configuration data.

    * Normalize all structs into python types.
    * Leave restructuring to later api calls, based on use case.
  """

  def map_loadfrom_datavalues(map, *keys):
    """
      Helper function:
      * Copy in data from `@ytt/data`.
      * Normalize results.
    """
    for key in keys:
      value = getattr(data.values, key)
      
      #! Ensures entire tree is decoded, not just top node.
      value = struct.decode(value)

      #! @TODO do this based on usage. (do we need singleton defaults?)
      value = normalize_to_list(value)

      #! replace source entry.
      map[key] = value
    end
  end

  #! @return fresh copy of data
  r = {}
  map_loadfrom_datavalues( r 
    , VOLUME_DETAILS
    , VOLUME_GROUPS
    , ENVIRONMENT_VALUES
    , ENVIRONMENT_GROUPS
    , SERVICE_PROFILES
  )
  return r
end

def _data_load_postprocess(data):
  """
    Transformations to be applied after a fresh data load.

    This should be about normalization, not anchor expansion.
  """

  def _process_volume_details():
    """
      Process the volume groups:
      * Allow for compose style string definitions.
      * Convert any `host/guest` definitions to keyed style.
      * Store compose ready results at `.definitions`.
    """
    for volume in data['volume-details']:
      volume['definitions'] = volumedetails_get_definitions(None, volume)
    end
  end

  def _process_volume_groups():
    """
      Process volume groups:
      * *! Current calculations moved to on demand helper functions. !*
    """ 
  end

  def _process_environment_groups():
    """
      Process environment groups:
      *! Current resolutions moved to on demand helper functions. *!
    """ 
  end

  def _process_service_profiles():
    """
      Process service-profiles:
      * calculate full extends path for easy traversals.
    """
    profiles = data[SERVICE_PROFILES]
    for profile in profiles:

      def extend_chain(profile, *keys):
        if not 'extend' in profile: return keys
        
        next_extend = profile['extend']

        next_profile = list_finddict_byname(profiles,next_extend)

        next_list = [k for k in keys]
        next_list.append(next_extend)
                
        return extend_chain(next_profile, *next_list)
      end
      
      profile[SERVICE_PROFILE.EXTEND_PATH] = extend_chain(profile)
    end
  end

  _process_volume_details()
  _process_service_profiles()
  
  #! Now Empty:
  _process_volume_groups()
  _process_environment_groups()
end

def _context():
  """
    Load and post process the configuration data.

    This method may or may not create a mutable shared context later.
  """

  #! load data.
  r = _data_load()

  #! post-process
  _data_load_postprocess(r)

  return r
end

def dc2():
  """
    docker-compose-config
    Function to build the main object.
  """
  context=_context()

  def _context_cache(context):
    return context
  end
  
  return struct.make_and_bind(
    context
  , fileversion=_composefile_version
  , services=_services
  , service=_service
  , profile=_profile
  , context=_context_cache
  )
end

#! --- UTILS ----- !#

def volume_string(*args):
  """ 
  Creates a docker volume string from the host and guest paths. 

  * (map.value) : use the value. Allows for literal with metadata.
  * (host, guest) : two strings to make path
  * ({host:_, guest:_}): two strings from map.
  """
  
  def from_paths(host, guest):
    return host + ":" + guest
  end

  if type(args[0]) == 'string': 
    if len(args) == 1: return args[0]
    if (len(args) == 2) & (type(args[1]) == 'string'): return from_paths(args[0],args[1])
  end

  if type(args[0]) == 'dict':
    map = args[0]
    if 'value' in map: return map['value']
    return from_paths(**map)
  end
end

def normalize_to_list(data):
  """
    @TODO: Normalize to support both types of configuration (list and dicts).
    
    * some data is an array of dicts(name=_;...)
    * some is just dicts.
    * This method should convert dicts into an array where each element is the nested `(dict).name=key`.
  """
  data_type = type(data)

  if data_type == 'list':
    return data
  elif data_type == 'dict':
    r = [ {'name':key} for key in data.keys()]
    for item in r:
      item.update(**data[item['name']])
    end
    return r
  end

  return [{'name': None, 'value': data}]
end

def list_finddicts_byname(list, *names):
  """
    Find all items in the list with the given name.
  """
  r = []
  for item in list:
    if type(item) == 'dict':
      if 'name' in item:
        for name in names:
          if name == item['name']: 
            r.append(item)
          end
        end
      end
    end
  end
  return r
end

def list_finddict_byname(list, name):
  """
    Find the first item in the list with the given name property.
  """
  for item in list:
    if type(item) == 'dict':
      if 'name' in item:
        if name == item['name']: return item
      end
    end
  end
  return None
end

def ifname_replacewithlookup(context, section, item):
  """
    If `item` is just a string id, pull the data from the context.
    Use this method to share algorithm between references and direct access.
  """
  if type(item) != 'string': return item

  return list_finddict_byname(context[section], item)
end

def volumegroup_get_volumedetails(context, volumegroup):
  """
    For the `volume-group`, get the volume details entries.
  """
  volumegroup = ifname_replacewithlookup(context, VOLUME_GROUPS, volumegroup)
  
  names = [name for name in volumegroup['volume-names']]
  r = list_finddicts_byname(context[VOLUME_DETAILS], *names)
  return r
end

def volumedetails_get_definitions(context, *volumedetails_list):
  """
    For the `volume-details`[], get a flattened array of volume definitions.
  """
  definitions = []
  
  for volumedetails in volumedetails_list:
    volumedetails = ifname_replacewithlookup(context, VOLUME_DETAILS, volumedetails)
    mappings = volumedetails[VOLUME_DETAIL.MAPPINGS]
    
    mappings_type = type(mappings)

    if (mappings_type == 'dict') or (mappings_type == 'string'): 
      mappings = [mappings]
    end
  
    for map in mappings:
      definitions.append( volume_string(map) )
    end
  end  
  
  return definitions
end

def environmentgroup_get_definitions(context, *enviromentgroup_list):
   """
    Get all of the definitions for one or more environment groups.
   """

  def resolve_dynamic(map):
    """
      Helper Function:
      * resolve 'use-value' keys.
    """
    if ENVIRONMENT_GROUP.USE_VALUE in map:
      use_value = map[ENVIRONMENT_GROUP.USE_VALUE]
      environmentvalue = list_finddict_byname(context[ENVIRONMENT_VALUES], use_value)
      return environmentvalue['value']
    end
    return str(map)
  end
  
  definitions=[]
  
  for environmentgroup in enviromentgroup_list:
    environmentgroup = ifname_replacewithlookup(context, ENVIRONMENT_GROUPS, environmentgroup)
    key_values = environmentgroup[ENVIRONMENT_GROUP.KEY_VALUES]
    
    for key, value in key_values.items():
      if type(value) == 'dict': 
        use_value = resolve_dynamic(value)
      elif type(value == 'string'):
        use_value = value
      end
      
      definitions.append(key+"="+use_value)
    end
  end
  
  return definitions
end


def elvis_key(context, key):
  """
    Get the value if the key is in the context, else an empty array.
  """
  if key in context: 
    return context[key]
  end
  return []
end

#!
#! API keys:
#!
SERVICE_PROFILES='service-profiles'
ENVIRONMENT_GROUPS='environment-groups'
ENVIRONMENT_VALUES='environment-values'
VOLUME_GROUPS='volume-groups'
VOLUME_DETAILS='volume-details'

ENVIRONMENT_GROUP=struct.make(
  KEY_VALUES='key-values'
, USE_VALUE='use-value'
)

SERVICE_PROFILE=struct.make(
  PROPERTIES="properties"  
, EXTEND_PATH='extend-path'
, VOLUME_GROUPS=VOLUME_GROUPS
, ENVIRONMENT_GROUPS=ENVIRONMENT_GROUPS
)

VOLUME_GROUP=struct.make(
  VOLUME_NAMES='volume-names'
)

VOLUME_DETAIL=struct.make(
  MAPPINGS='mappings'
, DEFINITIONS='definitions'
)

data-values-header.yml

---
#!
#! This example breaks down the section seperator for `@ytt:data` nodes.
#!
#! Following this example, you can keep multiple data sections in the same file, or split the files up.
#!
#! You cannot have any document separators after the `#@data/values` tag, unless you apply both annotations, as in this example.
#! 
---

#! Must have overlay library for tree merging
#@ load("@ytt:overlay", "overlay")

#! v= Ensure separation from previous content by placing separator above annotation. 
---
#! Annotations:
#! * Use as data/value node.
#@data/values
#! * Merge on all content types?
#@overlay/match by=overlay.all
#! * Treat all children as mergeable.
#@overlay/match-child-defaults missing_ok=True
---
test: null

data.dc2.yml

#!```ytt
#! Load the `compose` helper module, and initialize.
#@ load("compose.star", "dc2")
#!```

#! The dc2 (docker-compose compose) context.
#! These are the post processed results of the configuration loaded by `@ytt:data`.
dc2-context: #@ dc2().context()

data.docker-compose.yml

#@ load("@ytt:overlay", "overlay")

---
#!
#! `volume-details` defines a directory of docker volume definitions.
#!
#! * Items are keyed by `name`.
#! * Paths are declared separately as `mapping.host` and `mapping.guest`.
#! * Calculated `volume-definition` is inferred by api.
#! * Volumes are declared discretely, then referenced individually or joined into groups.
---
#@data/values
#@overlay/match by=overlay.all
#@overlay/match-child-defaults missing_ok=True
---
volume-details: 

#! Utility directories

 - name: host-dir
   description: |
     Containers can see the working directory of the host folder.
   mappings: ./.:/host/
   #! host: ./
   #! guest: /host/
      
 - name: utility-scripts
   description: |
     Shared scripts folder for all projects.
   mappings:
    - host: ./scripts
      guest: /scripts/
    
 - name: bin-mounts
   description: |
     Executables mapped into `/usr/bin/`.
   mappings:
    - host: ./bin/ytt-linux
      guest: /usr/bin/ytt
    - "./bin/composer.phar:/usr/bin/composer"
   
#! Mysql directories

 - name: mysql-data
   description: |
     The data folder for the mysql service.
   mappings:
    host: ./.volumes/mysql/
    guest: /var/lib/mysql

#! Wordpress: Application Directories

 - name: wordpress-data
   description: |
     The root folder of the wordpress application.
   mappings:
    host: ./.volumes/wp-app-data
    guest: /var/www/html

 - name: wordpress-cli-data
   description: |
     The user folder where wp-cli packages are installed.
   mappings:
    host: ./.volumes/wp-cli-data
    guest: /home/www-data/.wp-cli

#! Wordpress: Theme dev files. (Overlayed into mapped root)

 - name: wp-dev-themes
   description: |
     The host folder where theme files are stored.
   mappings:
    host: ./wp-dev/themes
    guest: /var/www/html/wp-content/themes
---


---
#!
#! `volume-groups` defines a directory of volume groups/profiles that can be resolved into a service profile definition.
#!
#! * actual volume mappings are referred to by the key `name` in `volume-details`.
---
#@data/values
#@overlay/match by=overlay.all
#@overlay/match-child-defaults missing_ok=True
---
volume-groups:

  - name: project-aware
    description: |
      Utility Group: 
      * peek out into `/host/` folder; reach any other part of the project by guest cli.
      * shared `/scripts/` folder; share `.sh` scripts across all projects..
      * bin mounts; share `/usr/bin` files across all projects.
    volume-names: 
      - host-dir
      - utility-scripts
      - bin-mounts

  - name: wordpress-and-friends
    description: | 
      Joined profile for wp/wp-cli/wp-dev.
    volume-names: 
      - wordpress-data
      - wordpress-cli-data
      - wp-dev-themes

  - name: mysql-data
    description: |
      The data files for the mysql backend.
    volume-names:
      - mysql-data
---


---
#!
#! `environment-values` defines a directory of environment property values (not keys), with attached metadata.
#!
#! * The values may have the same or multiple keys depending on target image, but the config value is static.
#! * @TODO simplify groups by having profiles here, for target environment types.
#!
---
#@data/values
#@overlay/match by=overlay.all
#@overlay/match-child-defaults missing_ok=True
---
environment-values:
  - name: mysql-root-password
    description: |
      The root password for the mysql database.
    value: example
---


---
#!
#! `environment-groups` defines a group of environment key/values.
#!
#! * They can be applied as a whole to a service profile.
#! * Values can be programatically decided using nested keys as commands.
#!
---
#@data/values
#@overlay/match by=overlay.all
#@overlay/match-child-defaults missing_ok=True
---
environment-groups:

  wordpress:
    description: |
      Env configurations for the wordpress app (db connection params, salting info, etc.)
    TODO: Bring in other wordpress configs. 
    key-values:
      WORDPRESS_PASSWORD: {use-value: mysql-root-password}

  mysql:
    description: |
      Env configuration for the mysql db.
    key-values: 
      MYSQL_ROOT_PASSWORD: {use-value: mysql-root-password}

---


---
#!
#! `service-profiles` defines the templates for services.
#!
#! These can be instantiatied at runtime, and inherit properties from other profiles.
#!
---
#@data/values
#@overlay/match by=overlay.all
#@overlay/match-child-defaults missing_ok=True
---
service-profiles:


  wp-base:
    extend: default
    description: |
      The base profile for wordpress images.
    environment-groups: 
      - wordpress
    volume-groups: 
      - wordpress-and-friends
  
  wp-app:
    extend: wp-base
    description: |
      The wordpress hosting application (php, wp-content/content)
    properties:
      image: wordpress:latest
      ports:
       - "8080:80"
    
  wp-cli:
    extend: wp-base
    description: |
      The `wp` cli application (php, wp-content/config, wp-cli in `/usr/bin`).
    properties:
      image: wordpress:cli
      tty: true
    
  wp-db:
    extend: default
    description: |
      The database backend for wordpress.
    environment-groups: 
      - mysql
    volume-groups: 
      - mysql-data
    properties:
      image: mysql:latest 
---


---
#@data/values
#@overlay/match by=overlay.all
#@overlay/match-child-defaults missing_ok=True
---
service-profiles:

  default:
    description: |
      Project aware image configuration.
    volume-groups:
      - project-aware
---

data.values.yml

#@ load("@ytt:data", "data")

data.values: #@ data.values

docker-compose.2.yml

#!```ytt
#! Load the `compose` helper module, and initialize.
#@ load("compose.star", "dc2")
#@ compose=dc2()
#!```

version: #@ compose.fileversion()

services: #@ compose.services("wp-db", "wp-app", "wp-cli")

docker-compose.yml

#!```ytt
#! Load the `compose` helper module, and initialize.
#@ load("compose.star", "dc2")
#@ compose=dc2()
#!```

version: #@ compose.fileversion()

services:

  mysql: #@ compose.service('wp-db')
    
  wordpress: #@ compose.service('wp-app')
    
  wp-cli: #@ compose.service('wp-cli')
    

funcs.star

load("@ytt:struct", "struct")
def process_volumes(volumes):
  r = []
  
  #! data module returns structs.
  #! struct needs to be decoded to a collection.
  decoded = struct.decode(volumes)
  
  r.append('TYPE ' + type(decoded))
  
  #! decoded data is assumed to be list
  #! a.k.a `data.volumes`
  for d in decoded:
    #! collect node to return value
    #! before inner keys are modified
    r.append(d)
    
    #! With the mapping node...
    mapping = d['mapping'] 
    
    #! Normalize the data (array|single)=>(array)
    map_type = type(mapping)
    if (map_type == 'list'): #! Array of entries.
      paths = mapping
    elif (map_type == 'dict'): #! Single entries are maps.
      paths = [mapping]
    end
   
    #! Create calculated field (volume string)
    for p in paths:
      p['volume'] = volume_string(p)
    end
    
    #! replace original node with normalized/calculated node
    d['mapping'] = paths
  end
  
  return r
end

def volume_string(*args):
  """ 
  Creates a docker volume string from the host and guest
  paths. Accepts two strings (host, guest) or a dict containing them.
  """
  
  def from_map(map):
    #! `**kwargs` expansion converts map to named params.
    return from_paths(**map)
  end
  
  def from_paths(host, guest):
    return host + ":" + guest
  end
  
  if len(args) == 2:
    return from_paths(args[0], args[1])
  end
  return from_map(args[0])
end