Deploy custom Ingress on GKE

app-deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-deployment
spec:
  selector:
    matchLabels:
      app: nginx
  replicas: 2
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx:latest
        ports:
        - name: http
          containerPort: 80
---
apiVersion: cloud.google.com/v1
kind: BackendConfig
metadata:
  name: nginx-bc
spec:
  cdn:
    enabled: true
    cacheMode: USE_ORIGIN_HEADERS
    cachePolicy:
      includeHost: true
      includeProtocol: false
      includeQueryString: true
    negativeCaching: false
---
apiVersion: v1
kind: Service
metadata:
  name: nginx-service
  annotations:
    cloud.google.com/backend-config: '{"default": "nginx-bc"}'
    cloud.google.com/neg: '{"ingress": true}'
spec:
  type: NodePort
  ports:
  - port: 80
    targetPort: 80
  selector:
    app: nginx

app-ingress.yaml

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: nginx-ingress
spec:
  rules:
  - host: ""
    http:
      paths:
      - path: /*
        pathType: ImplementationSpecific
        backend:
          service:
            name: nginx-service
            port:
              number: 80

default-http-backend.yaml.tpl

apiVersion: apps/v1
kind: Deployment
metadata:
  name: l7-default-backend
  namespace: kube-system
  labels:
    k8s-app: glbc
    kubernetes.io/name: "GLBC"
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: glbc
  template:
    metadata:
      labels:
        k8s-app: glbc
        name: glbc
    spec:
      containers:
      - name: default-http-backend
        # Any image is permissible as long as:
        # 1. It serves a 404 page at /
        # 2. It serves 200 on a /healthz endpoint
        image: k8s.gcr.io/defaultbackend-amd64:1.5
        livenessProbe:
          httpGet:
            path: /healthz
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 30
          timeoutSeconds: 5
        ports:
        - containerPort: 8080
        resources:
          limits:
            cpu: 10m
            memory: 20Mi
          requests:
            cpu: 10m
            memory: 20Mi
---
apiVersion: v1
kind: Service
metadata:
  # This must match the --default-backend-service argument of the l7 lb
  # controller and is required because GCE mandates a default backend.
  name: default-http-backend
  namespace: kube-system
  labels:
    k8s-app: glbc
    kubernetes.io/name: "GLBCDefaultBackend"
spec:
  # The default backend must be of type NodePort.
  type: NodePort
  ports:
  - port: 80
    targetPort: 8080
    protocol: TCP
    name: http
    nodePort: [NODE_PORT]
  selector:
    k8s-app: glbc

gce.conf.tpl

[global]
token-url = nil
# Your cluster's project
project-id = [PROJECT]
# Your cluster's network
network-name =  [NETWORK]
# Your cluster's subnetwork
subnetwork-name = [SUBNETWORK]
# Prefix for your cluster's IG
node-instance-prefix = gke-[CLUSTER_NAME]
# Network tags for your cluster's IG
node-tags = [NETWORK_TAGS]
# Zone the cluster lives in
local-zone = [ZONE]

glbc.yaml.tpl

apiVersion: apps/v1
kind: Deployment
metadata:
  name: l7-lb-controller
  namespace: kube-system
  annotations:
    scheduler.alpha.kubernetes.io/critical-pod: ''
  labels:
    k8s-app: gcp-lb-controller
    kubernetes.io/name: "GLBC"
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: gcp-lb-controller
  template:
    metadata:
      labels:
        k8s-app: gcp-lb-controller
        name: gcp-lb-controller
    spec:
      serviceAccountName: glbc
      terminationGracePeriodSeconds: 600
      hostNetwork: true
      containers:
      - image: [IMAGE_URL]
        livenessProbe:
          httpGet:
            path: /healthz
            port: 8086
            scheme: HTTP
          initialDelaySeconds: 30
          # healthz reaches out to GCE
          periodSeconds: 30
          timeoutSeconds: 15
          successThreshold: 1
          failureThreshold: 5
        name: l7-lb-controller
        volumeMounts:
        - mountPath: /etc/gce/
          name: gce-config-volume
        - mountPath: /var/secrets/google
          name: google-cloud-key
        env:
        - name: GOOGLE_APPLICATION_CREDENTIALS
          value: /var/secrets/google/key.json
        resources:
          # Request is set to accommodate this pod alongside the other
          # master components on a single core master.
          # TODO: Make resource requirements depend on the size of the cluster
          requests:
            cpu: 10m
            memory: 50Mi
        command:
        - /glbc
        - -v2
        - --config-file-path=/etc/gce/gce.conf
        - --healthz-port=8086
        - --logtostderr
        - --sync-period=600s
        - --gce-ratelimit=ga.Operations.Get,qps,10,100
        - --gce-ratelimit=alpha.Operations.Get,qps,10,100
        - --gce-ratelimit=ga.BackendServices.Get,qps,1.8,1
        - --gce-ratelimit=ga.HealthChecks.Get,qps,1.8,1
        - --gce-ratelimit=alpha.HealthChecks.Get,qps,1.8,1
        - --enable-backendconfig-healthcheck
      volumes:
      - name: google-cloud-key
        secret:
          secretName: glbc-gcp-key
      - name: gce-config-volume
        configMap:
          name: gce-config
          items:
          - key: gce.conf
            path: gce.conf

rbac.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: glbc
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: system:controller:glbc
  namespace: kube-system
  labels:
    addonmanager.kubernetes.io/mode: Reconcile
rules:
- apiGroups: [""]
  resources: ["configmaps"]
  verbs: ["get", "list", "watch", "update", "create", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: system:controller:glbc
  namespace: kube-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: system:controller:glbc
subjects:
- kind: ServiceAccount
  name: glbc
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: system:controller:glbc
rules:
- apiGroups: [""]
  resources: ["secrets"]
  verbs: ["get"]
- apiGroups: [""]
  resources: ["events"]
  verbs: ["get", "list", "watch", "update", "create", "patch"]
- apiGroups: [""]
  resources: ["endpoints", "services", "pods", "nodes", "namespaces"]
  verbs: ["get", "list", "watch"]
- apiGroups: [""]
  resources: ["services/status"]
  verbs: ["patch"]
- apiGroups: ["networking.istio.io"]
  resources: ["destinationrules"]
  verbs: ["get", "list", "watch", "update", "patch"]
- apiGroups: ["extensions", "networking.k8s.io"]
  resources: ["ingresses"]
  verbs: ["get", "list", "watch"]
- apiGroups: ["extensions", "networking.k8s.io"]
  resources: ["ingresses/status"]
  verbs: ["patch"]
# GLBC ensures that the `cloud.google.com/backendconfigs` CRD exists in a desired state:
# https://github.com/kubernetes/ingress-gce/blob/4918eb2f0f484f09ac9e5a975907a9b16ed2b344/cmd/glbc/main.go#L93
# TODO(rramkumar1): https://github.com/kubernetes/ingress-gce/issues/744
- apiGroups: ["apiextensions.k8s.io"]
  resources: ["customresourcedefinitions"]
  verbs: ["get", "list", "watch", "update", "create", "patch"]
- apiGroups: ["cloud.google.com"]
  resources: ["backendconfigs"]
  verbs: ["get", "list", "watch", "update", "create", "patch"]
# GLBC ensures that the `networking.gke.io/frontendconfigs` CRD exists and reconciles the configuration
# https://github.com/kubernetes/ingress-gce/blob/v1.9.4/cmd/glbc/main.go#L118
- apiGroups: ["networking.gke.io"]
  resources: ["frontendconfigs"]
  verbs: ["get", "list", "watch", "update", "create", "patch"]
- apiGroups: ["networking.gke.io"]
  resources: ["servicenetworkendpointgroups","gcpingressparams"]
  verbs: ["get", "list", "watch", "update", "create", "patch", "delete"]
- apiGroups: ["networking.k8s.io"]
  resources: ["ingressclasses"]
  verbs: ["get", "list", "watch", "update", "create", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: system:controller:glbc
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:controller:glbc
subjects:
- kind: ServiceAccount
  name: glbc
  namespace: kube-system

theia.yaml

kind: ServiceAccount
apiVersion: v1
metadata:
  name: ingress-e2e-test
  namespace: default
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: ingress-e2e-test
rules:
   - apiGroups: ["*"]
     resources: ["*"]
     verbs: ["*"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: ingress-e2e-test
subjects:
  - kind: ServiceAccount
    name: ingress-e2e-test
    namespace: default
roleRef:
  kind: ClusterRole
  name: ingress-e2e-test
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: pvc-theia
spec:
  storageClassName: "premium-rwo"
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: theia-golang
spec:
  replicas: 1
  selector:
    matchLabels:
      app: theia-golang
  template:
    metadata:
      labels:
        app: theia-golang
    spec:
      securityContext:
        fsGroup: 1001
      containers:
      - name: theia-golang
        image: gabihodoroaga/theia-golang:1.13-alpine
        ports:
          - name: ide
            containerPort: 3000
        env:
        - name: GOOGLE_APPLICATION_CREDENTIALS
          value: /var/secrets/google/key.json
        volumeMounts:
          - mountPath: /var/secrets/google
            name: google-cloud-key
          - mountPath: /home/project
            name: project-folder
      serviceAccount: ingress-e2e-test
      volumes:
      - name: google-cloud-key
        secret:
          secretName: glbc-gcp-key
      - name: project-folder
        persistentVolumeClaim:
          claimName: pvc-theia
---
apiVersion: v1
kind: Service
metadata:
  name: theia-golang-service
spec:
  type: NodePort
  ports:
  - name: ide
    port: 3000
    targetPort: 3000
  selector:
    app: theia-golang