OSA repo mirroring

ansible-role-requirements.yml

- name: apt-mirror
  scm: git
  src: http://github.com/Logan2211/ansible-apt-mirror
  version: master

repo-apt-mirror.yml

---

- name: Install apt mirrors to repo
  hosts: repo_all[0]
  user: root
  roles:
    - { role: "apt-mirror", tags: apt-mirror }
  vars:
    apt_mirror_user: nginx
    apt_mirror_group: www-data
    apt_mirror_base_path: /var/www/repo/apt
    apt_mirror_repos:
      - { url: "http://download.ceph.com/debian-{{ ceph_stable_release | default('jewel') }}/" }
      - { url: "https://mirror.rackspace.com/mariadb/repo/10.0/ubuntu" }
      - { url: "http://ppa.launchpad.net/vbernat/haproxy-1.5/ubuntu" }
      - { url: "https://repo.percona.com/apt" }
      - { url: "http://ppa.launchpad.net/project-calico/{{ calico_apt_repo_branch }}/ubuntu" }
      - { url: "http://ppa.launchpad.net/cz.nic-labs/bird/ubuntu" }
      - { url: "http://packages.elasticsearch.org/elasticsearch/1.5/debian", distribution: stable }
      - { url: "http://packages.elasticsearch.org/logstash/1.4/debian", distribution: stable }
      - { url: "http://repo.zabbix.com/zabbix/{{ zabbix_version | default('3.0') }}/ubuntu" }
      - { url: "https://repos.influxdata.com/ubuntu", components: "{{ influxdb_install_version | default('stable') }}" }

- name: Install apt keys to repo
  tags: repo-apt-keys
  hosts: repo_all[0]
  user: root
  pre_tasks:
    - name: Ensure apt-key destination directory exists
      file:
        path: "{{ key_dir }}"
        state: directory
    - name: Ensure gnupg package is installed
      apt:
        update_cache: yes
        cache_valid_time: 600
        name: "{{ item }}"
        state: present
      with_items:
        - gnupg
  tasks:
    - name: Install apt-keys to repo keyring
      shell: gpg
        --no-default-keyring
        --primary-keyring "{{ keyring_file | quote }}"
        --keyserver "{{ (item.keyserver | default(primary_keyserver)) | quote }}"
        --recv-keys "{{ item.hash_id }}"
      register: add_keys
      until: add_keys|success
      ignore_errors: True
      retries: 5
      delay: 2
      changed_when: "'imported' in add_keys.stderr"
      with_items: mirror_keys
      tags:
        - apt-key-mirror
        - apt-key-mirror-import
    - name: Export apt-keys to repo
      shell: gpg
        --no-default-keyring
        --primary-keyring "{{ keyring_file | quote }}"
        --export --armor "{{ item.hash_id }}"
        > "{{ key_dir | quote }}/{{ item.key_name | quote }}.asc"
      with_items: mirror_keys
      tags:
        - apt-key-mirror
        - apt-key-mirror-export
  vars:
    keyring_file: /tmp/repo.gpg
    key_dir: "{{ repo_service_home_folder }}/repo/apt-keys"
    primary_keyserver: "{{ keyserver_default_primary | default('hkp://keyserver.ubuntu.com:80') }}"
    fallback_keyserver: "{{ keyserver_default_fallback | default('hkp://p80.pool.sks-keyservers.net:80') }}"
    mirror_keys:
      - key_name: ceph
        hash_id: '0xe84ac2c0460f3994'
      - key_name: haproxy
        hash_id: '0xcffb779aadc995e4f350a060505d97a41c61b9cd'
      - key_name: mariadb
        hash_id: '0xcbcb082a1bb943db'
      - key_name: percona-xtrabackup
        hash_id: '0x1c4cbdcdcd2efd2a'
      - key_name: projectcalico
        hash_id: '0x67d7e35b3d40a6a7'
      - key_name: bird
        hash_id: '0xf2331238f9c59a45'
      - key_name: elasticsearch
        hash_id: '0x46095acc8548582c1a2699a9d27d666cd88e42B4'
      - key_name: zabbix
        hash_id: '79EA5ED4'
      - key_name: influxdb
        hash_id: '2582E0C5'

repo-misc-mirror.yml

---

- name: Mirror upstream items to repo
  hosts: repo_all[0]
  user: root
  tasks:
    - name: Ensure repo LXC cache directory exists
      file:
        path: "{{ repo_service_home_folder }}/repo/mirror"
        state: directory
        owner: "{{ repo_service_user_name }}"
        mode: "0755"
      tags:
        - repo-mirror
    - name: Cache Modern PIP on repo
      get_url:
        url: "{{ repo_get_pip_url }}"
        dest: "{{ repo_service_home_folder }}/repo/mirror/get-pip.py"
        force: "yes"
        validate_certs: "yes"
      register: get_pip
      until: get_pip | success
      ignore_errors: True
      retries: 5
      delay: 2
      tags:
        - repo-mirror-pip-install-script
    - name: Get Modern PIP on repo using fallback URL
      get_url:
        url: "{{ repo_get_pip_fallback_url }}"
        dest: "{{ repo_service_home_folder }}/repo/mirror/get-pip.py"
        force: "yes"
        validate_certs: "yes"
      when: get_pip | failed
      register: get_pip_fallback
      until: get_pip_fallback | success
      retries: 5
      delay: 2
      tags:
        - repo-mirror-pip-install-script
    - name: Mirror misc .deb files to repo
      get_url:
        url: "{{ item.url }}"
        dest: "{{ repo_service_home_folder }}/repo/mirror"
        mode: "0644"
        sha256sum: "{{ item.sha256 }}"
      with_items: repo_files_mirror
      register: package_download
      retries: 3
      delay: 10
      until: package_download|success
      tags:
        - repo-mirror-rabbitmq-package-deb
    - name: Mirror ceph-dash git repo
      git:
        repo: "{{ ceph_dash_repo }}"
        dest: "{{ repo_service_home_folder }}/repo/openstackgit/ceph-dash"
        clone: yes
        update: yes
        version: "{{ ceph_dash_version }}"
      tags:
        - repo-mirror-ceph-dash
  vars:
    ceph_dash_repo: https://github.com/Crapworks/ceph-dash
    ceph_dash_version: master
    repo_pip_upstream_url: https://bootstrap.pypa.io
    repo_get_pip_url: "{{ repo_pip_upstream_url }}/get-pip.py"
    repo_get_pip_fallback_url: https://raw.githubusercontent.com/pypa/get-pip/master/get-pip.py

user_repos.yml

## Cinder service
cinder_git_repo: https://user:pass@git.internal/cinder.git
cinder_git_package_name: cinder
cinder_git_install_branch: <internal_branch_sha>

## Horizon service
horizon_git_repo: https://user:pass@git.internal/horizon.git
horizon_git_package_name: horizon
horizon_git_install_branch: <internal_branch_sha>

## Neutron service
neutron_git_repo: https://user:pass@git.internal/neutron.git
neutron_git_package_name: neutron
neutron_git_install_branch: <internal_branch_sha>

## Nova service
nova_git_repo: https://user:pass@git.internal/nova.git
nova_git_package_name: nova
nova_git_install_branch: <internal_branch_sha>

user_variables.yml

#configure local file mirrors on repo
rabbitmq_upstream_url: https://www.rabbitmq.com/releases/rabbitmq-server/v3.5.7/rabbitmq-server_3.5.7-1_all.deb
xtrabackup_upstream_url: https://www.percona.com/downloads/XtraBackup/Percona-XtraBackup-2.2.13/binary/debian/trusty/x86_64/percona-xtrabackup-22_2.2.13-1.trusty_amd64.deb
kibana_upstream_url: https://download.elasticsearch.org/kibana/kibana/{{ kibana_version }}.tar.gz
repo_files_mirror:
  - name: rabbitmq
    url: "{{ rabbitmq_upstream_url }}"
    sha256: b8a42321c2f2689dc579911fbb583bd9c4d2ce2f20003d7050d5a324a6d2de42
  - name: xtrabackup
    url: "{{ xtrabackup_upstream_url }}"
    sha256: 2f58eedefa905583f0650f77bb2b149139c4066c7fb690202124fe5c7ac83e9e
  - name: kibana
    url: "{{ kibana_upstream_url }}"
    sha256: 480562733c2c941525bfa26326b6fae5faf83109b452a6c4e283a5c37e3086ee

#set items to local repo as repo-misc-mirror.yml play caches them for us.
pip_upstream_repo_url: "{{ openstack_repo_url }}/mirror"
rabbitmq_package_url: "{{ openstack_repo_url }}/mirror/{{ rabbitmq_upstream_url | basename }}"
galera_package_url: "{{ openstack_repo_url }}/mirror/{{ xtrabackup_upstream_url | basename }}"

apt_mirror_base_url: "{{ openstack_repo_url }}/apt/mirror"
ceph_apt_repo_url: "{{ apt_mirror_base_url }}/download.ceph.com/debian-{{ ceph_stable_release }}" #for osa ceph-client role
ceph_stable_repo: "{{ ceph_apt_repo_url }}" #for ceph-ansible upstream roles
galera_apt_repo_url: "{{ apt_mirror_base_url }}/mirror.rackspace.com/mariadb/repo/10.0/ubuntu"
galera_apt_percona_xtrabackup_url: "{{ apt_mirror_base_url }}/repo.percona.com/apt"
galera_client_apt_repo_url: "{{ galera_apt_repo_url }}"
haproxy_apt_repo_url: "http://{{ hostvars[groups['pkg_repo'][0]]['ansible_ssh_host'] }}:8181/apt/mirror/ppa.launchpad.net/vbernat/haproxy-1.5/ubuntu"
zabbix_apt_repo_url: "{{ apt_mirror_base_url }}/repo.zabbix.com/zabbix/{{ zabbix_version | default('3.0') }}/ubuntu"
influxdb_apt_repo_url: "{{ apt_mirror_base_url }}/repos.influxdata.com/ubuntu"

# Ceph, galera, haproxy apt-keys
ceph_gpg_keys:
  - key_name: 'ceph'
    url: "{{ apt_keys_url }}/ceph.asc"
    hash_id: '0xe84ac2c0460f3994'
galera_gpg_keys:
  - key_name: 'mariadb'
    url: "{{ apt_keys_url }}/mariadb.asc"
    hash_id: '0xcbcb082a1bb943db'
  - key_name: 'percona-xtrabackup'
    url: "{{ apt_keys_url }}/percona-xtrabackup.asc"
    hash_id: '0x1c4cbdcdcd2efd2a'
galera_client_gpg_keys: "{{ galera_gpg_keys }}"
haproxy_gpg_keys:
  - key_name: 'haproxy'
    url: "{{ apt_keys_url }}/haproxy.asc"
    hash_id: '0xcffb779aadc995e4f350a060505d97a41c61b9cd'
zabbix_gpg_keys:
  - key_name: 'zabbix'
    url: "{{ apt_keys_url }}/zabbix.asc"
    hash_id: '79EA5ED4'
influxdb_apt_key_url: "{{ apt_keys_url }}/influxdb.asc"