mertyildiran/29-mizutest-amqp-py-dep.yaml

## 29-mizutest-amqp-py-dep.yaml
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: mizutest-amqp-py
  labels:
    name: mizutest-amqp-py
  namespace: sock-shop
spec:
  replicas: 1
  selector:
    matchLabels:
      name: mizutest-amqp-py
  template:
    metadata:
      labels:
        name: mizutest-amqp-py
    spec:
      containers:
      - name: mizutest-amqp-py
        image: mertyildiran/mizutest-amqp-py:latest
        env:
        - name: PYTHONUNBUFFERED
          value: "1"
        - name: PYTHONIOENCODING
          value: "UTF-8"
        imagePullPolicy: Always
        command: ["python3", "example.py"]
        resources:
          limits:
            cpu: 200m
            memory: 200Mi
          requests:
            cpu: 100m
            memory: 100Mi
        ports:
        - containerPort: 5672
        securityContext:
          runAsNonRoot: true
          runAsUser: 10001
          capabilities:
            drop:
              - all
            add:
              - NET_BIND_SERVICE
          readOnlyRootFilesystem: true
      nodeSelector:
        beta.kubernetes.io/os: linux

## 30-mizutest-amqp-py-svc.yaml
---
apiVersion: v1
kind: Service
metadata:
  name: mizutest-amqp-py
  annotations:
        prometheus.io/scrape: 'true'
  labels:
    name: mizutest-amqp-py
  namespace: sock-shop
spec:
  ports:
    # the port that this service should serve on
  - port: 5672
    targetPort: 5672
  selector:
    name: mizutest-amqp-py

## 31-kafka-zookeeper-dep.yaml
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: zookeeper
  name: zookeeper
  namespace: sock-shop
spec:
  replicas: 1
  selector:
    matchLabels:
      app: zookeeper
  template:
    metadata:
      labels:
        app: zookeeper
    spec:
      containers:
      - image: library/zookeeper:3.4.13
        imagePullPolicy: IfNotPresent
        name: zookeeper
        ports:
        - containerPort: 2181
        env:
        - name: ZOO_MY_ID
          value: "1"

## 32-kafka-zookeeper-svc.yaml
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: zookeeper-service
  name: zookeeper-service
  namespace: sock-shop
spec:
  type: NodePort
  ports:
  - name: zookeeper-port
    port: 2181
    # nodePort: 30181
    targetPort: 2181
  selector:
    app: zookeeper

## 33-kafka-borker-stfs.yaml
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: kafka
  namespace: sock-shop
spec:
  selector:
    matchLabels:
      app: kafka
  serviceName: "kafka"
  replicas: 3
  podManagementPolicy: OrderedReady
  template:
    metadata:
      labels:
        app: kafka # has to match .spec.selector.matchLabels
    spec:
      containers:
      - name: kafka
        image: wurstmeister/kafka:2.11-2.0.0
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 9092
          name: plaintext
        - containerPort: 9999
          name: jmx
        env:
          - name: KAFKA_ADVERTISED_PORT
            value: "9092"
          - name: BROKER_ID_COMMAND
            value: "hostname | cut -d'-' -f2"
          - name: KAFKA_ZOOKEEPER_CONNECT
            value: "zookeeper-service:2181"
          - name: KAFKA_LISTENERS
            value: "PLAINTEXT://:9092"
          - name: KAFKA_JMX_OPTS
            value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=9999 -Dcom.sun.management.jmxremote.rmi.port=9999 -Djava.rmi.server.hostname=127.0.0.1"
          - name: JMX_PORT
            value: "9999"

## 34-kafka-broker-svc.yaml
---
apiVersion: v1
kind: Service
metadata:
  name: kafka
  namespace: sock-shop
  labels:
    app: kafka
spec:
  ports:
  - port: 9092
    name: plaintext
  - port: 9999
    name: jmx
  clusterIP: None
  selector:
    app: kafka

## 35-mizutest-kafka-py-dep.yaml
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: mizutest-kafka-py
  labels:
    name: mizutest-kafka-py
  namespace: sock-shop
spec:
  replicas: 1
  selector:
    matchLabels:
      name: mizutest-kafka-py
  template:
    metadata:
      labels:
        name: mizutest-kafka-py
    spec:
      containers:
      - name: mizutest-kafka-py
        image: mertyildiran/mizutest-kafka-py:latest
        imagePullPolicy: Always
        command: ["./run.sh"]
        resources:
          limits:
            cpu: 200m
            memory: 200Mi
          requests:
            cpu: 100m
            memory: 100Mi
        ports:
        - containerPort: 9092
        securityContext:
          runAsNonRoot: true
          runAsUser: 10001
          capabilities:
            drop:
              - all
            add:
              - NET_BIND_SERVICE
          readOnlyRootFilesystem: true
      nodeSelector:
        beta.kubernetes.io/os: linux

## 36-mizutest-kafka-py-svc.yaml
---
apiVersion: v1
kind: Service
metadata:
  name: mizutest-kafka-py
  annotations:
        prometheus.io/scrape: 'true'
  labels:
    name: mizutest-kafka-py
  namespace: sock-shop
spec:
  ports:
    # the port that this service should serve on
  - port: 9092
    targetPort: 9092
  selector:
    name: mizutest-kafka-py

## 37-mizutest-kafka-go-dep.yaml
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: mizutest-kafka-go
  labels:
    name: mizutest-kafka-go
  namespace: sock-shop
spec:
  replicas: 1
  selector:
    matchLabels:
      name: mizutest-kafka-go
  template:
    metadata:
      labels:
        name: mizutest-kafka-go
    spec:
      containers:
      - name: mizutest-kafka-go
        image: mertyildiran/mizutest-kafka-go:latest
        imagePullPolicy: Always
        command: ["./run.sh"]
        resources:
          limits:
            cpu: 200m
            memory: 200Mi
          requests:
            cpu: 100m
            memory: 100Mi
        ports:
        - containerPort: 9092
        # securityContext:
        #   runAsNonRoot: true
        #   runAsUser: 10001
        #   capabilities:
        #     drop:
        #       - all
        #     add:
        #       - NET_BIND_SERVICE
        #   readOnlyRootFilesystem: true
      nodeSelector:
        beta.kubernetes.io/os: linux

## 38-mizutest-kafka-go-svc.yaml
---
apiVersion: v1
kind: Service
metadata:
  name: mizutest-kafka-go
  annotations:
        prometheus.io/scrape: 'true'
  labels:
    name: mizutest-kafka-go
  namespace: sock-shop
spec:
  ports:
    # the port that this service should serve on
  - port: 9092
    targetPort: 9092
  selector:
    name: mizutest-kafka-go

## README.md

      
    Raw
  

              README.md
            
          
    Mizu message queue capture on Kubernetes

We will make additions to Sock Shop microservice demo application to demonstrate message queue capture of Mizu.
Place the Kubernetes manifests that are provided by this Gist into https://github.com/microservices-demo/microservices-demo/tree/master/deploy/kubernetes/manifests
Run the command below to update your cluster:
$ kubectl create -f deploy/kubernetes/manifests/00-sock-shop-ns.yaml -f deploy/kubernetes/manifests
Set agent-image: mertyildiran/mizuagent:latest in your Mizu config.yaml or use your own image.
Your cluster should look like this:
$ kubectl get pod -A
NAMESPACE              NAME                                         READY   STATUS    RESTARTS   AGE
sock-shop              carts-b4d4ffb5c-gbdmf                        1/1     Running   0          71m
sock-shop              carts-db-6c6c68b747-vj2p7                    1/1     Running   0          71m
sock-shop              catalogue-759cc6b86-rp7hl                    1/1     Running   0          71m
sock-shop              catalogue-db-96f6f6b4c-4dg48                 1/1     Running   0          71m
sock-shop              front-end-5c89db9f57-w6xhc                   1/1     Running   0          71m
sock-shop              kafka-0                                      1/1     Running   0          71m
sock-shop              kafka-1                                      1/1     Running   0          71m
sock-shop              kafka-2                                      1/1     Running   0          71m
sock-shop              mizutest-amqp-py-68b4c64478-fpf26            1/1     Running   0          40m
sock-shop              mizutest-kafka-py-665b5489c-5zjkg            1/1     Running   0          40m
sock-shop              orders-7664c64d75-99lkn                      1/1     Running   0          71m
sock-shop              orders-db-659949975f-22k5v                   1/1     Running   0          71m
sock-shop              payment-7bcdbf45c9-vsltj                     1/1     Running   0          71m
sock-shop              queue-master-5f6d6d4796-5fhxt                1/1     Running   0          71m
sock-shop              rabbitmq-5bcbb547d7-vd289                    2/2     Running   0          71m
sock-shop              session-db-7cf97f8d4f-jlm46                  1/1     Running   0          71m
sock-shop              shipping-7f7999ffb7-qt2rs                    1/1     Running   0          71m
sock-shop              user-68df64db9c-lm9f6                        1/1     Running   0          71m
sock-shop              user-db-6df7444fc-vf4bc                      1/1     Running   0          71m
sock-shop              zookeeper-7b69697846-5qhbw                   1/1     Running   0          71m

AMQP

See if the pod is printing messages into stdout:
$ kubectl logs --follow mizutest-amqp-py-68b4c64478-fpf26 -n sock-shop
-----
1
Declared queue
Declared exchange
Queue bind
Basic consume
Basic publish
Basic publish
-----
2
Declared queue
Declared exchange
Queue bind
Basic consume
Basic publish
...
The mizutest-amqp-py deployment constantly produces and consumes messages from RabbitMQ to generate message queue traffic.
Now tap into the pod:
$ ./cli/bin/mizu__ tap mizutest-amqp-py-68b4c64478-fpf26 -n sock-shop
Expect a few seconds delay for messages to show up.
Kafka

See if the pod is printing messages into stdout:
$ kubectl logs --follow mizutest-kafka-py-665b5489c-5zjkg -n sock-shop
-----
CreateTopic
Failed to create topic topic1: KafkaError{code=TOPIC_ALREADY_EXISTS,val=36,str="Topic 'topic1' already exists."}
Failed to create topic topic2: KafkaError{code=TOPIC_ALREADY_EXISTS,val=36,str="Topic 'topic2' already exists."}
Produce
Produce
Produce
Message delivered to mytopic [0]
Message delivered to mytopic [0]
Message delivered to mytopic [0]
-----
CreateTopic
Failed to create topic topic1: KafkaError{code=TOPIC_ALREADY_EXISTS,val=36,str="Topic 'topic1' already exists."}
Failed to create topic topic2: KafkaError{code=TOPIC_ALREADY_EXISTS,val=36,str="Topic 'topic2' already exists."}
Produce
Produce
...
The mizutest-kafka-py deployment constantly produces and consumes messages from Apache Kafka to generate message queue traffic.
Now tap into the pod:
./cli/bin/mizu__ tap mizutest-kafka-py-665b5489c-5zjkg -n sock-shop
Expect a few seconds delay for messages to show up.
Note: Sock Shop produces an AMQP message when an order is placed from orders service and consumes it from shipping service as far as I understand. queue-master service is also involved some how. Other than that, Sock Shop doesn't use message queues as a message BUS between microservices but rather uses it as a job queue. I couldn't really trigger that single message queue usage in Sock Shop from the UI. Therefore I added a bunch of deployments to make it active in terms of message queues.
	---
	apiVersion: apps/v1
	kind: Deployment
	metadata:
	name: mizutest-amqp-py
	labels:
	name: mizutest-amqp-py
	namespace: sock-shop
	spec:
	replicas: 1
	selector:
	matchLabels:
	name: mizutest-amqp-py
	template:
	metadata:
	labels:
	name: mizutest-amqp-py
	spec:
	containers:
	- name: mizutest-amqp-py
	image: mertyildiran/mizutest-amqp-py:latest
	env:
	- name: PYTHONUNBUFFERED
	value: "1"
	- name: PYTHONIOENCODING
	value: "UTF-8"
	imagePullPolicy: Always
	command: ["python3", "example.py"]
	resources:
	limits:
	cpu: 200m
	memory: 200Mi
	requests:
	cpu: 100m
	memory: 100Mi
	ports:
	- containerPort: 5672
	securityContext:
	runAsNonRoot: true
	runAsUser: 10001
	capabilities:
	drop:
	- all
	add:
	- NET_BIND_SERVICE
	readOnlyRootFilesystem: true
	nodeSelector:
	beta.kubernetes.io/os: linux
	---
	apiVersion: v1
	kind: Service
	metadata:
	name: mizutest-amqp-py
	annotations:
	prometheus.io/scrape: 'true'
	labels:
	name: mizutest-amqp-py
	namespace: sock-shop
	spec:
	ports:
	# the port that this service should serve on
	- port: 5672
	targetPort: 5672
	selector:
	name: mizutest-amqp-py
	---
	apiVersion: apps/v1
	kind: StatefulSet
	metadata:
	name: kafka
	namespace: sock-shop
	spec:
	selector:
	matchLabels:
	app: kafka
	serviceName: "kafka"
	replicas: 3
	podManagementPolicy: OrderedReady
	template:
	metadata:
	labels:
	app: kafka # has to match .spec.selector.matchLabels
	spec:
	containers:
	- name: kafka
	image: wurstmeister/kafka:2.11-2.0.0
	imagePullPolicy: IfNotPresent
	ports:
	- containerPort: 9092
	name: plaintext
	- containerPort: 9999
	name: jmx
	env:
	- name: KAFKA_ADVERTISED_PORT
	value: "9092"
	- name: BROKER_ID_COMMAND
	value: "hostname \| cut -d'-' -f2"
	- name: KAFKA_ZOOKEEPER_CONNECT
	value: "zookeeper-service:2181"
	- name: KAFKA_LISTENERS
	value: "PLAINTEXT://:9092"
	- name: KAFKA_JMX_OPTS
	value: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.port=9999 -Dcom.sun.management.jmxremote.rmi.port=9999 -Djava.rmi.server.hostname=127.0.0.1"
	- name: JMX_PORT
	value: "9999"