videobug-server-deployment-gcp

GCP-DEPLOYMENT-README.md

Service account HMAC key generation

1. create a new service account
2. with perission roles/storage.objectAdmin (access to google cloud storage)
3. generate HMAC keys

gcloud iam service-accounts create videobug-service-account --display-name "videobug service account"
gcloud projects add-iam-policy-binding $(gcloud config get-value project)  --member="serviceAccount:videobug-service-account@$(gcloud config get-value project).iam.gserviceaccount.com"  --role="roles/storage.objectAdmin"


gsutil hmac create videobug-service-account@$(gcloud config get-value project).iam.gserviceaccount.com

Access ID:   GOOG1EK3UCPL6K6OJB6LNGM7DSFU4U22SPVGHQTIV4KXXXXXXXXXXXXXXXXXX
Secret:      XXX/OSxxQ0xxxxJ0WxRTxx9YFx5xx0AxF1SPxxxx

Set your project as default in google storage interopability (open in new tab and check under heading "Default project for interoperable access")

https://console.cloud.google.com/storage/settings;tab=interoperability?project=

Deployment

Download videobug.jinja

wget https://gist.githubusercontent.com/artpar/1c471c722c81877f60d4e99337e92fd1/raw/47cb80b1034c2e3de443f8a4847ecb6179198630/videobug.jinja

Execute

gcloud deployment-manager deployments create videobug --template=videobug.jinja --properties="zone:'us-west1-a',region:'us-west1',accessKey:'<ACCESS-KEY>',secretKey:'<SECRET-KEY>'"

Get Public IP

gcloud compute addresses describe  --global videobug-public-ip

address: 34.117.44.161
addressType: EXTERNAL
creationTimestamp: '2022-05-31T23:44:53.432-07:00'
description: ''
id: '5991168970906579978'
kind: compute#address
name: videobug-public-ip
networkTier: PREMIUM
selfLink: https://www.googleapis.com/compute/v1/projects/shining-camp-344911/global/addresses/videobug-public-ip
status: IN_USE
users:
- https://www.googleapis.com/compute/v1/projects/shining-camp-344911/global/forwardingRules/videobug-http

videobug.jinja

{% set project = env["project"] %}
{% set deployment = env["deployment"] %}
{% set accessKey = properties["accessKey"] %}
{% set secretKey = properties["secretKey"] %}
{% set region = properties["region"] %}
{% set name = env["deployment"] %}
{% set instanceName = "%s-vm" % deployment %}
{% set bucketName = "%s-%s-%s" % (name, project, deployment) %}
{% set zone = properties["zone"] %}
{% set machineType = "e2-standard-4" %}
{% set networks = [] %}
{% set aerospikeNodes = ["1", "2", "3"] %}
{% set videobugNodes = ["1", "2"] %}
{% set subnetworks = properties["subnetwork"] %}
{% set externalIPs = properties["externalIP"] %}
{% set bootDiskType = properties["bootDiskType"] %}
{% set bootDiskSizeGb = properties["bootDiskSizeGb"] %}

resources:
  - name: videobug-network
    type: compute.v1.network
    properties:
      name: videobug-network
      autoCreateSubnetworks: false
  - name: videobug-subnet
    type: compute.v1.subnetwork
    properties:
      name: videobug-subnet
      network: $(ref.videobug-network.selfLink)
      ipCidrRange: 10.138.0.0/16
      region: {{ region }}
  - name: videobug-server-group
    type: compute.v1.instanceGroup
    properties:
      name: videobug-server-group
      network: $(ref.videobug-network.selfLink)
      subnetwork: $(ref.videobug-subnet.selfLink)
      region: {{ region }}
      zone: {{ zone }}
      namedPorts:
        - name: videobug-server-port
          port: 80
        - name: videobug-server-https
          port: 443
  - name: videobug-router
    type: compute.v1.router
    properties:
      region: {{ region }}
      nats:
        - name: videobug-nat-config
          natIpAllocateOption: AUTO_ONLY
          sourceSubnetworkIpRangesToNat: ALL_SUBNETWORKS_ALL_IP_RANGES
          region: {{ region }}
      network: $(ref.videobug-network.selfLink)
  - name: videobug-firewall
    type: compute.beta.firewall
    properties:
      name: videobug-allow-lb-and-healthcheck
      network: $(ref.videobug-network.selfLink)
      sourceRanges:
        - 130.211.0.0/22
        - 35.191.0.0/16
      allowed:
        - IPProtocol: tcp
          ports:
            - 80
            - 443
  - name: videobug-iap-firewall
    type: compute.beta.firewall
    properties:
      name: videobug-iap-ssh
      network: $(ref.videobug-network.selfLink)
      sourceRanges:
        - 35.235.240.0/20
      allowed:
        - IPProtocol: tcp
          ports:
            - 22
  - name: videobug-aerospike-firewall
    type: compute.beta.firewall
    properties:
      name: videobug-aerospike-ports
      network: $(ref.videobug-network.selfLink)
      sourceRanges:
        - 10.138.0.0/16
      allowed:
        - IPProtocol: tcp
          ports:
            - 3000
            - 3001
            - 3002
            - 3003
{% for node in aerospikeNodes %}
  - name: {{ name }}-aerospike-{{ node }}
    type: compute.v1.instance
    properties:
      instanceName: {{ name }}-aerospike-{{ node }}
      sourceImage: projects/debian-cloud/global/images/debian-10-buster-v20220519
      zone: {{ zone }}
      machineType: zones/{{ zone }}/machineTypes/{{ machineType }}
      networks:
        - projects/{{ project }}/global/networks/videobug-network
      subnetworks:
        - videobug-subnet
      bootDiskType: projects/{{ project }}/zones/us-west4-b/diskTypes/pd-ssd
      bootDiskSizeGb: 10
      canIpForward: false
      serviceAccounts:
        - email: default
          scopes:
            - 'https://www.googleapis.com/auth/cloud.useraccounts.readonly'
            - 'https://www.googleapis.com/auth/devstorage.read_only'
            - 'https://www.googleapis.com/auth/logging.write'
            - 'https://www.googleapis.com/auth/monitoring.write'
            - 'https://www.googleapis.com/auth/trace.append'
      tags:
        items:
          - {{ env["deployment"] }}-deployment
          - aerospike-server
      networkInterfaces:
      - subnetwork: $(ref.videobug-subnet.selfLink)
        networkIP: 10.138.0.1{{ node }}
      metadata:
        items:
          - key: startup-script
            value: |
              #!/bin/bash
              curl -o aerospike.tgz https://download.aerospike.com/artifacts/aerospike-server-community/6.0.0.1/aerospike-server-community-6.0.0.1-debian10.tgz
              tar -xvf aerospike.tgz
              cd aerospike-server-community-6.0.0.1-debian10
              sudo ./asinstall
              echo c2VydmljZSB7DQogICAgICAgIHByb3RvLWZkLW1heCAxNTAwMA0KfQ0KDQpsb2dnaW5nIHsNCiAgICAgICAgY29uc29sZSB7DQogICAgICAgICAgICAgICAgY29udGV4dCBhbnkgaW5mbw0KICAgICAgICB9DQp9DQoNCm5ldHdvcmsgew0KICAgICAgICBzZXJ2aWNlIHsNCiAgICAgICAgICAgICAgICBhZGRyZXNzIGFueQ0KICAgICAgICAgICAgICAgIHBvcnQgMzAwMA0KICAgICAgICB9DQoNCiAgICAgICAgaGVhcnRiZWF0IHsNCiAgICAgICAgICBtb2RlIG1lc2ggICAgICAgICAgICAgICAgICAgIyBTZW5kIGhlYXJ0YmVhdHMgdXNpbmcgTWVzaCAoVW5pY2FzdCkgcHJvdG9jb2wNCiAgICAgICAgICBhZGRyZXNzIDEwLjEzOC4wLjExICAgICAgICAgIyAoT3B0aW9uYWwpIChEZWZhdWx0OiBhbnkpIElQIG9mIHRoZSBOSUMgb24NCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIyB3aGljaCB0aGlzIG5vZGUgaXMgbGlzdGVuaW5nIHRvIGhlYXJ0YmVhdA0KICAgICAgICAgIHBvcnQgMzAwMiAgICAgICAgICAgICAgICAgICAjIHBvcnQgb24gd2hpY2ggdGhpcyBub2RlIGlzIGxpc3RlbmluZyB0bw0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAjIGhlYXJ0YmVhdA0KICAgICAgICAgIG1lc2gtc2VlZC1hZGRyZXNzLXBvcnQgMTAuMTM4LjAuMTEgMzAwMiAjIElQIGFkZHJlc3MgZm9yIHNlZWQgbm9kZSBpbiB0aGUgY2x1c3Rlcg0KICAgICAgICAgIG1lc2gtc2VlZC1hZGRyZXNzLXBvcnQgMTAuMTM4LjAuMTIgMzAwMiAjIElQIGFkZHJlc3MgZm9yIHNlZWQgbm9kZSBpbiB0aGUgY2x1c3Rlcg0KICAgICAgICAgIG1lc2gtc2VlZC1hZGRyZXNzLXBvcnQgMTAuMTM4LjAuMTMgMzAwMiAjIElQIGFkZHJlc3MgZm9yIHNlZWQgbm9kZSBpbiB0aGUgY2x1c3Rlcg0KDQogICAgICAgICAgaW50ZXJ2YWwgMTUwICAgICAgICAgICAgICAgICMgTnVtYmVyIG9mIG1pbGxpc2Vjb25kcyBiZXR3ZWVuIGhlYXJ0YmVhdHMNCiAgICAgICAgICB0aW1lb3V0IDEwICAgICAgICAgICAgICAgICAgIyBOdW1iZXIgb2YgaGVhcnRiZWF0IGludGVydmFscyB0byB3YWl0IGJlZm9yZQ0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAjIHRpbWluZyBvdXQgYSBub2RlDQogICAgICAgIH0NCg0KICAgICAgICBmYWJyaWMgew0KICAgICAgICAgICAgICAgIHBvcnQgMzAwMQ0KICAgICAgICB9DQoNCiAgICAgICAgaW5mbyB7DQogICAgICAgICAgICAgICAgcG9ydCAzMDAzDQogICAgICAgIH0NCn0NCg0KDQpuYW1lc3BhY2UgdmlkZW9idWcgew0KICAgIHJlcGxpY2F0aW9uLWZhY3RvciAyICAgICAjIEZvciBtdWx0aXBsZSBub2Rlcywga2VlcCAyIGNvcGllcyBvZiB0aGUgZGF0YQ0KICAgIHN0b3Atd3JpdGVzLXBjdCA5MCAgICAgICAjIFN0b3Agd3JpdGVzIGlmIGNhcGFjaXR5IGV4Y2VlZHMgOTAlIG9mIG1lbW9yeSBzaXplDQogICAgZGVmYXVsdC10dGwgMCAgICAgICAgICAgICMgV3JpdGVzIGZyb20gY2xpZW50IHRoYXQgZG8gbm90IHByb3ZpZGUgYSBUVEwNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAjIHdpbGwgZGVmYXVsdCB0byAwIG9yIG5ldmVyIGV4cGlyZQ0KICAgIG1lbW9yeS1zaXplIDUwRyAgICAgICAgICMgTWF4aW11bSBtZW1vcnkgYWxsb2NhdGlvbiBmb3IgcHJpbWFyeQ0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAjIGFuZCBzZWNvbmRhcnkgaW5kZXhlcy4NCiAgICBzdG9yYWdlLWVuZ2luZSBkZXZpY2UgeyAgICAgIyBDb25maWd1cmUgdGhlIHN0b3JhZ2UtZW5naW5lIHRvIHVzZSBwZXJzaXN0ZW5jZQ0KICAgICAgICBkZXZpY2UgL2Rldi9zZGIgICAgIyByYXcgZGV2aWNlLiBNYXhpbXVtIHNpemUgaXMgMiBUaUINCiAgICAgICAgd3JpdGUtYmxvY2stc2l6ZSAxMjhLICAgIyBhZGp1c3QgYmxvY2sgc2l6ZSB0byBtYWtlIGl0IGVmZmljaWVudCBmb3IgU1NEcy4NCiAgICB9DQp9DQo= | base64 -d > /etc/aerospike/aerospike.conf
              systemctl enable aerospike
              systemctl start aerospike
      disks:
        - kind: projects/{{ project }}/zones/{{ zone }}/diskTypes/pd-ssd
          boot: true
          autoDelete: true
          diskSizeGb: 10
          description: videobug-aerospike-bootdisk-{{ node }}
          initializeParams:
            diskName: videobug-aerospike-bootdisk-{{ node }}
            diskSizeGb: 10
            sourceImage: projects/debian-cloud/global/images/debian-10-buster-v20220519
            diskType: projects/{{ project }}/zones/{{ zone }}/diskTypes/pd-ssd
        - kind: projects/{{ project }}/zones/{{ zone }}/diskTypes/pd-ssd
          boot: false
          autoDelete: true
          diskSizeGb: 50
          description: videobug-aerospike-datadisk-{{ node }}
          initializeParams:
            diskName: videobug-aerospike-datadisk-{{ node }}
            diskSizeGb: 50
            diskType: projects/{{ project }}/zones/{{ zone }}/diskTypes/pd-ssd
{% endfor %}
{% for node in videobugNodes %}
  - name: {{ name }}-{{ node }}
    type: compute.v1.instance
    properties:
      instanceName: {{ name }}-{{ node }}
      zone: {{ zone }}
      machineType: zones/{{ zone }}/machineTypes/{{ machineType }}
      metadata:
        items:
          - key: startup-script
            value: |
              #!/bin/bash
              apt install snapd -y
              snap install videobug-server
              snap set videobug-server port=80
              snap set videobug-server profiles=local,aerospike,s3
              snap set videobug-server aerospike-hosts=10.138.0.11:3000
              snap set videobug-server s3-endpoint=https://storage.googleapis.com
              snap set videobug-server s3-bucketname={{ bucketName }}
              snap set videobug-server aerospike-namespace=videobug
              snap set videobug-server s3-access-key={{ accessKey }}
              snap set videobug-server s3-secret-key={{ secretKey }}
      networks:
        - projects/{{ project }}/global/networks/videobug-network
      subnetworks:
        - videobug-subnet
      bootDiskType: projects/{{ project }}/zones/us-west4-b/diskTypes/pd-ssd
      bootDiskSizeGb: 10
      canIpForward: false
      serviceAccounts:
        - email: default
          scopes:
            - 'https://www.googleapis.com/auth/cloud.useraccounts.readonly'
            - 'https://www.googleapis.com/auth/devstorage.read_only'
            - 'https://www.googleapis.com/auth/logging.write'
            - 'https://www.googleapis.com/auth/monitoring.write'
            - 'https://www.googleapis.com/auth/trace.append'
      tags:
        items:
          - {{ env["deployment"] }}-deployment
          - http-tag
          - http-server
          - https-server
      networkInterfaces:
      - subnetwork: $(ref.videobug-subnet.selfLink)
        networkIP: 10.138.0.2{{ node }}
      disks:
        - kind: projects/{{ project }}/zones/{{ zone }}/diskTypes/pd-ssd
          boot: true
          autoDelete: true
          diskSizeGb: 10
          description: videobug-boot-disk-{{ node }}
          initializeParams:
            diskName: videobug-disk-{{ node }}
            diskSizeGb: 10
            sourceImage: projects/debian-cloud/global/images/debian-11-bullseye-v20220519
            diskType: projects/{{ project }}/zones/{{ zone }}/diskTypes/pd-ssd
{% endfor %}
  - name: videobug-1-members
    action: gcp-types/compute-v1:compute.instanceGroups.addInstances
    properties:
      project: {{ project }}
      zone: {{ zone }}
      not: $(ref.videobug-server-group.selfLink)
      instanceGroup: $(ref.videobug-server-group.name)
      instances:
      {% for node in videobugNodes %}
        - instance: $(ref.{{ name }}-{{ node }}.selfLink)
      {% endfor %}
  - name: videobug-public-ip
    type: compute.v1.globalAddresses
    properties:
      name: videobug-public-ip
  - name: videobug-healthcheck
    type: compute.v1.healthCheck
    properties:
      checkIntervalSec: 1
      healthyThreshold: 1
      unhealthyThreshold: 2
      timeoutSec: 1
      type: HTTP
      httpHealthCheck:
        host: ''
        port: 80
        proxyHeader: NONE
        requestPath: /
  - name: videobug-backend
    type: compute.v1.backendServices
    properties:
      name: videobug-backend
      portName: videobug-server-port
      protocol: HTTP
      healthChecks:
        - $(ref.videobug-healthcheck.selfLink)
      backends:
        - description: videobug-server-backend
          group: $(ref.videobug-server-group.selfLink)
          balancingMode: UTILIZATION
          maxUtilization: 0.8
  - name: videobug-urlmap
    type: compute.v1.urlMaps
    properties:
      name: videobug-backend
      portName: videobug-server-port
      protocol: HTTP
      defaultService: $(ref.videobug-backend.selfLink)
  - name: videobug-proxy
    type: compute.v1.targetHttpProxies
    properties:
      name: videobug-proxy
      urlMap: $(ref.videobug-urlmap.selfLink)
  - name: videobug-http
    type: compute.v1.globalForwardingRules
    properties:
      name: videobug-http
      IPProtocol: TCP
      IPAddress: $(ref.videobug-public-ip.selfLink)
      loadBalancingScheme: EXTERNAL
      portRange: 80-80
      region: {{ region }}
      target: global/targetHttpProxies/$(ref.videobug-proxy.name)
      not: $(ref.videobug-proxy.selfLink)
  - name: {{ bucketName }}
    type: storage.v1.bucket
    properties:
      name: {{ bucketName }}
outputs:
  - name: deployment
    value: {{ deployment }}
  - name: project
    value: {{ project }}
  - name: public-ip
    value: $(ref.videobug-public-ip.address)