Recipes from here, cooked and baked by ssro
AWS environment, K3S kubernetes environment
Make sure that the instance used for this setup has Route53 permissions (proper instance role)
Can use persistent volumes or attached disk. In this case, there's a disk attached to the instance as /data and XFS formatted
After the setup is complete, use https://registry-1.domain.tld (please use your own domain) as your mirror (reconfigure docker daemons running in your infra to use this as mirror)
curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION="v1.23.3+k3s1" INSTALL_K3S_EXEC="server --write-kubeconfig-mode 644 --disable servicelb --disable traefik --kubelet-arg eviction-hard=memory.available<300Mi --kubelet-arg=image-gc-high-threshold=85 --kubelet-arg=image-gc-low-threshold=80 --kube-apiserver-arg=enable-aggregator-routing=true" sh -
kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.6.0/cert-manager.yaml
kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.11.0/manifests/namespace.yaml
kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.11.0/manifests/metallb.yaml
export IP=$(curl -s http://169.254.169.254/latest/meta-data/local-ipv4)
or
export IP=$(hostname -I | cut -d " " -f1)
tee << EOF | kubectl apply -f -
apiVersion: v1
kind: ConfigMap
metadata:
namespace: metallb-system
name: config
data:
config: |
address-pools:
- name: default
protocol: layer2
addresses:
- ${IP}/32
EOF
SOMERANDOMSTRING
registry-1.domain.tld
YOUR_EMAIL_ADDRESS
YOUR_REGION
YOUR_ZONE_ID_AS_IT_APPEARS_IN_ROUTE53
Apply ingress controller & patches
mkdir ~/manifests/ingress-nginx && cd ~/manifests/ingress-nginx
curl -o ingress-nginx-controller.yaml https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.1/deploy/static/provider/cloud/deploy.yaml
cat << EOF > cm-ingress-controller.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: ingress-nginx-controller
namespace: ingress-nginx
data:
allow-snippet-annotations: "true"
use-forwarded-headers: "true"
enable-brotli: "true"
hsts: "true"
hsts-include-subdomains: "true"
hsts-max-age: "63072000"
server-tokens: "false"
ssl-ciphers: ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
ssl-protocols: TLSv1.2 TLSv1.3
ssl-session-cache: "true"
ssl-session-cache-size: 20m
ssl-session-tickets: "false"
use-gzip: "true"
gzip-level: "3"
EOF
cat <<EOF > kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ingress-nginx-controller.yaml
patches:
- cm-ingress-controller.yaml
namespace: ingress-nginx
EOF
kubectl apply -k .Apply the registry manifest & dependencies:
apiVersion: v1
kind: Namespace
metadata:
name: registry
---
apiVersion: v1
kind: ConfigMap
metadata:
name: redis-configuration
namespace: registry
data:
redis-cfg: |-
maxmemory 1g
activedefrag yes
active-defrag-threshold-lower 10
active-defrag-threshold-upper 100
tcp-backlog 65535
maxmemory-policy allkeys-lru
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
set-max-intset-entries 512
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit replica 256mb 64mb 60
client-output-buffer-limit pubsub 256mb 64mb 60
stop-writes-on-bgsave-error no
---
apiVersion: v1
kind: Service
metadata:
labels:
app: redis
name: redis
namespace: registry
spec:
ports:
- port: 6379
selector:
app: redis
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: redis
name: redis
namespace: registry
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 1
selector:
matchLabels:
app: redis
strategy:
rollingUpdate:
maxSurge: 25%
maxUnavailable: 25%
type: RollingUpdate
template:
metadata:
labels:
app: "redis"
preset: "true"
spec:
serviceAccountName: redis
containers:
- image: redis:6.2.5-alpine
securityContext:
runAsNonRoot: true
runAsUser: 999
allowPrivilegeEscalation: false
imagePullPolicy: IfNotPresent
livenessProbe:
exec:
command:
- redis-cli
- ping
failureThreshold: 5
initialDelaySeconds: 30
timeoutSeconds: 5
name: redis
command: ["redis-server", "/usr/local/etc/redis/redis.conf"]
ports:
- containerPort: 6379
name: redis
readinessProbe:
exec:
command:
- redis-cli
- ping
initialDelaySeconds: 5
periodSeconds: 5
timeoutSeconds: 1
resources:
limits:
cpu: 1
memory: 1.5Gi
requests:
cpu: 200m
memory: 128Mi
volumeMounts:
- mountPath: /data
name: data
- mountPath: /usr/local/etc/redis
name: config
initContainers:
- command:
- sysctl
- -w
- net.core.somaxconn=65535
image: busybox:1.34.0
imagePullPolicy: IfNotPresent
name: somaxconn
securityContext:
privileged: true
- command:
- sysctl
- -w
- vm.overcommit_memory=1
image: busybox:1.34.0
imagePullPolicy: IfNotPresent
name: mem-ovrcommit
securityContext:
privileged: true
- command:
- sysctl
- -w
- net.ipv4.tcp_max_syn_backlog=65535
image: busybox:1.34.0
imagePullPolicy: IfNotPresent
name: syn-backlog
securityContext:
privileged: true
- name: disable-thp
image: busybox
command: ["sh", "-c"]
args:
- |-
set -e
set -o pipefail
echo never > /rootfs/sys/kernel/mm/transparent_hugepage/enabled
echo never > /rootfs/sys/kernel/mm/transparent_hugepage/defrag
grep -q -F [never] /sys/kernel/mm/transparent_hugepage/enabled
grep -q -F [never] /sys/kernel/mm/transparent_hugepage/defrag
securityContext:
privileged: true
volumeMounts:
- name: sys
mountPath: /rootfs/sys
volumes:
- emptyDir: {}
name: data
- name: config
configMap:
name: redis-configuration
items:
- key: redis-cfg
path: redis.conf
- name: sys
hostPath:
path: /sys
---
apiVersion: v1
kind: ConfigMap
metadata:
name: registry
namespace: registry
data:
registry-config.yml: |
version: 0.1
log:
fields:
service: registry
storage:
filesystem:
rootdirectory: /data/registry
cache:
blobdescriptor: redis
maintenance:
uploadpurging:
enabled: true
age: 120h
interval: 12h
dryrun: false
delete:
enabled: true
http:
addr: :5000
headers:
X-Content-Type-Options: [nosniff]
# Access-Control-Allow-Origin: ['*']
# Access-Control-Allow-Methods: ['HEAD', 'GET', 'OPTIONS']
# Access-Control-Max-Age: [1728000]
# Access-Control-Expose-Headers: ['Docker-Content-Digest']
secret: SOMERANDOMSTRING
draintimeout: 60s
redis:
addr: redis:6379
db: 0
dialtimeout: 10ms
readtimeout: 10ms
writetimeout: 10ms
pool:
maxidle: 16
maxactive: 64
idletimeout: 300s
health:
storagedriver:
enabled: true
interval: 10s
threshold: 3
proxy:
remoteurl: https://registry-1.docker.io
# username: [username]
# password: [password]
---
apiVersion: v1
kind: Service
metadata:
name: registry
namespace: registry
labels:
app: registry
spec:
selector:
app: registry
ports:
- port: 5000
name: http-5000
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: registry
namespace: registry
labels:
app: registry
spec:
progressDeadlineSeconds: 600
replicas: 1
selector:
matchLabels:
app: registry
revisionHistoryLimit: 3
strategy:
type: Recreate
template:
metadata:
labels:
app: registry
spec:
volumes:
- name: registry
hostPath:
path: /data/registry
- name: config
configMap:
name: registry
items:
- key: registry-config.yml
path: config.yml
containers:
- name: registry
image: registry:2.7.1
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5000
name: http-5000
protocol: TCP
resources:
limits:
cpu: 1
memory: 2Gi
requests:
cpu: 200m
memory: 128Mi
volumeMounts:
- name: config
mountPath: /etc/docker/registry
readOnly: true
- name: registry
mountPath: /data/registry
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: registry
namespace: registry
annotations:
cert-manager.io/cluster-issuer: "letsencrypt-prod"
nginx.ingress.kubernetes.io/proxy-body-size: "500m"
nginx.ingress.kubernetes.io/proxy-read-timeout: "3600"
nginx.ingress.kubernetes.io/proxy-send-timeout: "3600"
nginx.ingress.kubernetes.io/proxy-buffer-size: "256k"
nginx.ingress.kubernetes.io/proxy-next-upstream: "error timeout http_502"
nginx.ingress.kubernetes.io/http2-max-header-size: "256k"
nginx.ingress.kubernetes.io/large-client-header-buffers: "256 256k"
nginx.ingress.kubernetes.io/client-header-buffer-size: "256k"
nginx.ingress.kubernetes.io/client-body-buffer-size: "5m"
nginx.ingress.kubernetes.io/use-gzip: "true"
nginx.ingress.kubernetes.io/configuration-snippet: |
more_set_headers "X-Content-Type-Options: nosniff";
more_set_headers "X-Frame-Options: SAMEORIGIN";
more_set_headers "X-Xss-Protection: 1; mode=block";
more_set_headers "Content-Security-Policy: upgrade-insecure-requests";
more_set_headers "Referrer-Policy: no-referrer-when-downgrade";
spec:
ingressClassName: nginx
tls:
- secretName: tls-secret
hosts:
- registry-1.domain.tld
rules:
- host: registry-1.domain.tld
http:
paths:
- pathType: Prefix
path: "/"
backend:
service:
name: registry
port:
number: 5000
---
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
name: letsencrypt-prod
spec:
acme:
server: https://acme-v02.api.letsencrypt.org/directory
email: YOUR_EMAIL_ADDRESS
privateKeySecretRef:
name: letsencrypt-prod
solvers:
- selector: {}
dns01:
route53:
region: YOUR_REGION
hostedZoneID: YOUR_ZONE_ID_AS_IT_APPEARS_IN_ROUTE53
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: tls-secret
namespace: registry
spec:
secretName: tls-secret
issuerRef:
name: letsencrypt-prod
kind: ClusterIssuer
commonName: '*.domain.tld'
dnsNames:
- '*.domain.tld'
---