How to DESTROY your AWS services (and come back to life in minutes) V.1

experiment.json

  "version": "1.0.0",
  "title": "Terminate the database master should not prevent application from running",
  "description": "As we are relying on Patroni to provide HA PostgreSQL, we expect that the application should run fine even when the PostgreSQL master goes away",
  "tags": [
      "kubernetes",
      "postgresql",
      "cloudnative",
      "database"
  ],
  "steady-state-hypothesis": {
      "title": "Services are all available and healthy",
      "probes": [
          {
              "type": "probe",
              "name": "application-should-be-alive-and-healthy",
              "tolerance": true,
              "provider": {
                  "type": "python",
                  "module": "chaosk8s.probes",
                  "func": "microservice_available_and_healthy",
                  "arguments": {
                      "name": "frontend-app",
                      "ns": "default"
                  }
              }
          },
          {
              "type": "probe",
              "name": "application-must-respond",
              "tolerance": 200,
              "provider": {
                  "type": "http",
                  "verify_tls": false,
                  "url": "https://app.cosmos.foo/"
              }
          }
      ]
  },
  "method": [
      {
          "type": "action",
          "name": "terminate-db-master",
          "provider": {
              "type": "python",
              "module": "chaosk8s.pod.actions",
              "func": "terminate_pods",
              "arguments": {
                  "label_selector": "spilo-role=master",
                  "name_pattern": "frontend-db-[0-9]$",
                  "rand": true,
                  "ns": "default"
              }
          },
          "pauses": {
              "after": 2
          }
      },
      {
          "type": "probe",
          "ref": "application-must-respond"
      },
      {
          "type": "probe",
          "name": "fetch-application-logs",
          "provider": {
              "type": "python",
              "module": "chaosk8s.pod.probes",
              "func": "read_pod_logs",
              "arguments": {
                  "label_selector": "app=frontend-app",
                  "container_name": "frontend-app",
                  "last": "20s",
                  "ns": "default"
              }
          },
          "pauses": {
              "before": 5
          }
      },
      {
          "type": "probe",
          "name": "fetch-db-logs",
          "provider": {
              "type": "python",
              "module": "chaosk8s.pod.probes",
              "func": "read_pod_logs",
              "arguments": {
                  "label_selector": "application=spilo",
                  "last": "30s",
                  "ns": "default"
              }
          },
          "pauses": {
              "before": 20
          }
      },
      {
          "type": "probe",
          "name": "fetch-patroni-operator-logs",
          "provider": {
              "type": "python",
              "module": "chaosk8s.pod.probes",
              "func": "read_pod_logs",
              "arguments": {
                  "label_selector": "name=postgres-operator",
                  "last": "20s",
                  "ns": "default"
              }
          }
      }
  ],
  "rollbacks": []
}