Skip to content

Instantly share code, notes, and snippets.

@cludden

cludden/README.md

Last active December 18, 2019 01:01
Show Gist options
  • Save cludden/2fda07b5783f490baa05f61c54072ae4 to your computer and use it in GitHub Desktop.
Save cludden/2fda07b5783f490baa05f61c54072ae4 to your computer and use it in GitHub Desktop.
Download ZIP
argo
Raw
README.md

platform-argo

infrastructure component for managing argo and related resources

Getting Started

  1. prereqs
  1. terraform
# render terraform variables
$ jsonnet -m . teams.jsonnet

# initialize backend
$ terraform init

# apply
$ terraform apply --var-file terraform.auto.tfvars.json
  1. kubecfg
# apply k8s manifests
$ kubecfg update --gc-tag argo manifest.jsonnet
Raw
argo-events.yml
---
# All argo-events services are bound to the "argo-events" service account.
# In RBAC enabled setups, this SA is bound to specific roles.
apiVersion: v1
kind: ServiceAccount
metadata:
name: argo-events-sa
namespace: argo-events
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: argo-events-binding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: argo-events-role
subjects:
- kind: ServiceAccount
name: argo-events-sa
namespace: argo-events
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: argo-events-role
rules:
- apiGroups:
- apiextensions.k8s.io
- apiextensions.k8s.io/v1beta1
verbs:
- create
- delete
- deletecollection
- get
- list
- patch
- update
- watch
resources:
- customresourcedefinitions
- apiGroups:
- argoproj.io
verbs:
- create
- delete
- deletecollection
- get
- list
- patch
- update
- watch
resources:
- workflows
- workflows/finalizers
- gateways
- gateways/finalizers
- sensors
- sensors/finalizers
- apiGroups:
- ""
resources:
- pods
- pods/exec
- configmaps
- secrets
- services
- events
- persistentvolumeclaims
verbs:
- create
- get
- list
- watch
- update
- patch
- delete
- apiGroups:
- "batch"
resources:
- jobs
verbs:
- create
- get
- list
- watch
- update
- patch
- delete
- apiGroups:
- "apps/v1"
- "apps/v1beta2"
- "apps/v1beta1"
resources:
- deployments
verbs:
- create
- get
- list
- watch
- update
- patch
- delete
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
rbac.authorization.k8s.io/aggregate-to-admin: "true"
name: argo-events-aggregate-to-admin
rules:
- apiGroups:
- argoproj.io
resources:
- gateways
- gateways/finalizers
- sensors
- sensors/finalizers
verbs:
- create
- delete
- deletecollection
- get
- list
- patch
- update
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
rbac.authorization.k8s.io/aggregate-to-edit: "true"
name: argo-events-aggregate-to-edit
rules:
- apiGroups:
- argoproj.io
resources:
- gateways
- gateways/finalizers
- sensors
- sensors/finalizers
verbs:
- create
- delete
- deletecollection
- get
- list
- patch
- update
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
rbac.authorization.k8s.io/aggregate-to-view: "true"
name: argo-events-aggregate-to-view
rules:
- apiGroups:
- argoproj.io
resources:
- gateways
- gateways/finalizers
- sensors
- sensors/finalizers
verbs:
- get
- list
- watch
---
# Define a "sensor" custom resource definition
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: sensors.argoproj.io
spec:
group: argoproj.io
names:
kind: Sensor
listKind: SensorList
plural: sensors
singular: sensor
scope: Namespaced
version: v1alpha1
---
# Define a "gateway" custom resource definition
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: gateways.argoproj.io
spec:
group: argoproj.io
names:
kind: Gateway
listKind: GatewayList
plural: gateways
singular: gateway
scope: Namespaced
version: v1alpha1
---
# The sensor-controller configmap includes configuration information for the sensor-controller
# To watch sensors created in different namespace than the controller is deployed in, remove the namespace: argo-events.
# Similarly to watch sensors created in specific namespace, change to namespace: <your_namespace>
apiVersion: v1
kind: ConfigMap
metadata:
name: sensor-controller-configmap
data:
config: |
instanceID: argo-events
namespace: argo-events
---
# The gateway-controller configmap includes configuration information for the gateway-controller
# To watch gateways created in different namespace than the controller is deployed in, remove the namespace: argo-events.
# Similarly to watch gateways created in specific namespace, change to namespace: <your_namespace>
apiVersion: v1
kind: ConfigMap
metadata:
name: gateway-controller-configmap
data:
config: |
instanceID: argo-events
namespace: argo-events
---
# The sensor-controller listens for changes on the sensor CRD and creates sensor executor jobs
apiVersion: apps/v1
kind: Deployment
metadata:
name: sensor-controller
spec:
replicas: 1
selector:
matchLabels:
app: sensor-controller
template:
metadata:
labels:
app: sensor-controller
spec:
serviceAccountName: argo-events-sa
containers:
- name: sensor-controller
image: argoproj/sensor-controller
imagePullPolicy: Always
env:
- name: SENSOR_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: SENSOR_CONFIG_MAP
value: sensor-controller-configmap
---
# The gateway-controller listens for changes on the gateway CRD and creates gateway
apiVersion: apps/v1
kind: Deployment
metadata:
name: gateway-controller
spec:
replicas: 1
selector:
matchLabels:
app: gateway-controller
template:
metadata:
labels:
app: gateway-controller
spec:
serviceAccountName: argo-events-sa
containers:
- name: gateway-controller
image: argoproj/gateway-controller
imagePullPolicy: Always
env:
- name: GATEWAY_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: GATEWAY_CONTROLLER_CONFIG_MAP
value: gateway-controller-configmap
Raw
argo.yml
---
# This is an auto-generated file. DO NOT EDIT
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: workflows.argoproj.io
spec:
group: argoproj.io
names:
kind: Workflow
plural: workflows
shortNames:
- wf
scope: Namespaced
version: v1alpha1
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: argo
namespace: argo
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: argo-ui
namespace: argo
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
rbac.authorization.k8s.io/aggregate-to-admin: "true"
name: argo-aggregate-to-admin
rules:
- apiGroups:
- argoproj.io
resources:
- workflows
- workflows/finalizers
verbs:
- create
- delete
- deletecollection
- get
- list
- patch
- update
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
rbac.authorization.k8s.io/aggregate-to-edit: "true"
name: argo-aggregate-to-edit
rules:
- apiGroups:
- argoproj.io
resources:
- workflows
- workflows/finalizers
verbs:
- create
- delete
- deletecollection
- get
- list
- patch
- update
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
rbac.authorization.k8s.io/aggregate-to-view: "true"
name: argo-aggregate-to-view
rules:
- apiGroups:
- argoproj.io
resources:
- workflows
- workflows/finalizers
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: argo-cluster-role
rules:
- apiGroups:
- ""
resources:
- pods
- pods/exec
verbs:
- create
- get
- list
- watch
- update
- patch
- delete
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- watch
- list
- apiGroups:
- ""
resources:
- persistentvolumeclaims
verbs:
- create
- delete
- apiGroups:
- argoproj.io
resources:
- workflows
- workflows/finalizers
verbs:
- get
- list
- watch
- update
- patch
- delete
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: argo-ui-cluster-role
rules:
- apiGroups:
- ""
resources:
- pods
- pods/exec
- pods/log
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- apiGroups:
- argoproj.io
resources:
- workflows
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: argo-binding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: argo-cluster-role
subjects:
- kind: ServiceAccount
name: argo
namespace: argo
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: argo-ui-binding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: argo-ui-cluster-role
subjects:
- kind: ServiceAccount
name: argo-ui
namespace: argo
---
apiVersion: v1
kind: ConfigMap
metadata:
name: workflow-controller-configmap
namespace: argo
---
apiVersion: v1
kind: Service
metadata:
name: argo-ui
namespace: argo
spec:
ports:
- port: 80
targetPort: 8001
selector:
app: argo-ui
---
apiVersion: apps/v1beta2
kind: Deployment
metadata:
name: argo-ui
namespace: argo
spec:
selector:
matchLabels:
app: argo-ui
template:
metadata:
labels:
app: argo-ui
spec:
containers:
- env:
- name: ARGO_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: IN_CLUSTER
value: "true"
- name: ENABLE_WEB_CONSOLE
value: "false"
- name: BASE_HREF
value: /
image: argoproj/argoui:v2.4.0
name: argo-ui
serviceAccountName: argo-ui
---
apiVersion: apps/v1beta2
kind: Deployment
metadata:
name: workflow-controller
namespace: argo
spec:
selector:
matchLabels:
app: workflow-controller
template:
metadata:
labels:
app: workflow-controller
spec:
containers:
- args:
- --configmap
- workflow-controller-configmap
- --executor-image
- argoproj/argoexec:v2.2.1
command:
- workflow-controller
image: argoproj/workflow-controller:v2.2.1
name: workflow-controller
serviceAccountName: argo
Raw
main.tf
################################################################################
## Terraform
################################################################################
terraform {
required_version = "~> 0.12.6"
}
################################################################################
## Variables
################################################################################
variable "kubernetes_worker_role" {
description = "kubernetes worker role"
type = string
}
variable "region" {
description = "aws region"
type = string
}
variable "teams" {
description = "map of teams by name"
type = map(object({
name = string,
iam_policies = object({
managed = map(string),
custom = map(list(object({
actions = list(string),
effect = string,
resources = list(string),
conditions = list(object({
test = string,
variable = string,
value = list(any),
})),
}))),
}),
}))
}
################################################################################
## Constants
################################################################################
locals {
policies_custom = flatten([
for name, team in var.teams : [
for alias, statements in team.iam_policies.custom : {
id = "${name}-${alias}"
team = name,
alias = alias,
statements = statements
}
]
])
policies_managed = flatten([
for name, team in var.teams : [
for alias, arn in team.iam_policies.managed : {
id = "${name}-${alias}"
team = name,
alias = alias,
arn = arn
}
]
])
tags = {
BillingGroup = "v7"
component = "platform-argo"
terraform = "true"
workspace = terraform.workspace
}
}
################################################################################
## Providers
################################################################################
provider "aws" {
region = var.region
}
################################################################################
## Resources
################################################################################
# define trust policy
data "aws_iam_policy_document" "argo_trust_policy" {
statement {
actions = ["sts:AssumeRole"]
effect = "Allow"
principals {
type = "AWS"
identifiers = [var.kubernetes_worker_role]
}
principals {
type = "Service"
identifiers = ["ec2.amazonaws.com"]
}
}
}
# provision argo role for each team
resource "aws_iam_role" "argo" {
for_each = var.teams
name = "argo-${each.value.name}"
description = "argo workflow role for ${each.value.name} team"
assume_role_policy = data.aws_iam_policy_document.argo_trust_policy.json
tags = local.tags
}
# provision custom argo policies
resource "aws_iam_policy" "argo_custom" {
for_each = {
for policy in local.policies_custom : policy.id => policy
}
name = "argo-${each.value.id}"
description = "argo role policy for ${each.value.team} team"
policy = data.aws_iam_policy_document.argo_custom[each.value.id].json
}
# define custom argo policies
data "aws_iam_policy_document" "argo_custom" {
for_each = {
for policy in local.policies_custom : policy.id => policy
}
dynamic "statement" {
for_each = each.value.statements
content {
actions = statement.value.actions
effect = statement.value.effect
resources = statement.value.resources
dynamic "condition" {
for_each = statement.value.conditions
content {
test = condition.value.test
variable = condition.value.variable
values = condition.value.values
}
}
}
}
}
# attach custom argo policies to argo role
resource "aws_iam_role_policy_attachment" "argo_custom" {
for_each = {
for policy in local.policies_custom : policy.id => policy
}
role = aws_iam_role.argo[each.value.team].id
policy_arn = aws_iam_policy.argo_custom[each.value.id].arn
}
# attach managed argo policies to argo role
resource "aws_iam_role_policy_attachment" "argo_managed" {
for_each = {
for policy in local.policies_managed : policy.id => policy
}
role = aws_iam_role.argo[each.value.team].id
policy_arn = each.value.arn
}
################################################################################
## Outputs
################################################################################
output "roles" {
value = {
for team in var.teams : team.name => aws_iam_role.argo[team.name]
}
}
Raw
manifest.jsonnet
local kube = import 'https://raw.githubusercontent.com/bitnami-labs/kube-libsonnet/master/kube.libsonnet';
local kubecfg = import 'kubecfg.libsonnet';
local teams = import 'teams.jsonnet';
function(labels={})
kube.List() {
items: std.flattenArrays([
// import argo manifest
std.filterMap($.filterObjects, $.setNamespace('argo'), kubecfg.parseYaml(importstr 'argo.yml')),
// import argo-events manifest
std.filterMap($.filterObjects, $.setNamespace('argo-events'), kubecfg.parseYaml(importstr 'argo-events.yml')),
]) + std.flattenArrays([
// provision team-scoped resources
$.teamObjects(teams.data[teamName] { name: teamName })
for teamName in std.objectFields(teams.data)
]),
// valid k8s filter function
filterObjects(obj)::
obj != null && std.objectHas(obj, 'apiVersion') && std.objectHas(obj, 'kind'),
// factory function for namespace mapper fn
setNamespace(ns)::
function(obj)
if std.length(std.find(obj.kind, ['ClusterRole', 'ClusterRoleBinding', 'CustomResourceDefinition', 'Namespace'])) > 0 then
obj
else
obj {
metadata+: {
namespace: ns,
},
},
// generate team-scoped kubernetes objects
teamObjects(team)::
// create team-scoped argo namespace
local ns = kube.Namespace('argo-%s' % team.name);
// create workflow service account
local sa = kube.ServiceAccount('workflow') {
metadata+: {
namespace: ns.metadata.name,
},
};
// grant workflow service account rbac
local rb = kube.RoleBinding('workflow') {
metadata+: {
namespace: ns.metadata.name,
},
subjects_: [sa],
roleRef_: {
kind: 'ClusterRole',
metadata: {
name: 'edit',
},
},
};
[
ns,
sa,
rb,
],
}
Raw
teams.jsonnet
{
data:: {
'data-services': {
iam_policies: {
custom: {
test: [
{
actions: ['s3:ListBuckets'],
effect: 'Allow',
resources: ['*'],
conditions: [],
},
],
},
managed: {
AmazonAPIGatewayAdministrator: 'arn:aws:iam::aws:policy/AmazonAPIGatewayAdministrator',
},
},
},
sre: {
iam_policies: {
managed: {
AdministratorAccess: 'arn:aws:iam::aws:policy/AdministratorAccess',
},
},
},
},
'terraform.auto.tfvars.json': {
kubernetes_worker_role: 'test',
region: 'us-east-1',
teams: std.mapWithKey(function(key, value) value {
name: key,
iam_policies+: {
custom+: {},
managed+: {},
},
}, $.data),
},
}
Raw
terraform.auto.tfvars.json
{
"kubernetes_worker_role": "arn:aws:iam::301764520140:role/data-services-terraform-root",
"region": "us-east-1",
"teams": {
"data-services": {
"iam_policies": {
"custom": {
"test": [
{
"actions": [
"s3:ListBuckets"
],
"conditions": [ ],
"effect": "Allow",
"resources": [
"*"
]
}
]
},
"managed": {
"AmazonAPIGatewayAdministrator": "arn:aws:iam::aws:policy/AmazonAPIGatewayAdministrator"
}
},
"name": "data-services"
},
"sre": {
"iam_policies": {
"custom": { },
"managed": {
"AdministratorAccess": "arn:aws:iam::aws:policy/AdministratorAccess"
}
},
"name": "sre"
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment