From 25d505161f86660d5b2adefd23e5666abc7b0837 Mon Sep 17 00:00:00 2001 From: Tom Hutter Date: Sun, 21 Nov 2021 20:14:02 +0100 Subject: [PATCH] Changed k8s installation files for a bit more sophisticated setup. --- docs/install/k8s/10-configmap.yaml | 7 +- docs/install/k8s/15-secrets.yaml | 10 ++ docs/install/k8s/20-service-account.yml | 5 + docs/install/k8s/30-pv.yaml | 50 --------- docs/install/k8s/30-pvc.yaml | 33 +----- docs/install/k8s/40-sts-postgresql.yaml | 142 +++++++++++++++++++++++ docs/install/k8s/45-service-db.yaml | 19 ++++ docs/install/k8s/50-deployment.yaml | 143 +++++++++++++++++++----- docs/install/k8s/60-service.yaml | 7 ++ docs/install/k8s/70-ingress.yaml | 38 +++++++ docs/install/kubernetes.md | 103 ++++++++++++++--- 11 files changed, 427 insertions(+), 130 deletions(-) create mode 100644 docs/install/k8s/15-secrets.yaml create mode 100644 docs/install/k8s/20-service-account.yml delete mode 100644 docs/install/k8s/30-pv.yaml create mode 100644 docs/install/k8s/40-sts-postgresql.yaml create mode 100644 docs/install/k8s/45-service-db.yaml create mode 100644 docs/install/k8s/70-ingress.yaml diff --git a/docs/install/k8s/10-configmap.yaml b/docs/install/k8s/10-configmap.yaml index 8a939d08..181f8066 100644 --- a/docs/install/k8s/10-configmap.yaml +++ b/docs/install/k8s/10-configmap.yaml @@ -4,12 +4,14 @@ metadata: labels: app: recipes name: recipes-nginx-config + namespace: default data: nginx-config: |- events { worker_connections 1024; } http { + include mime.types; server { listen 80; server_name _; @@ -24,10 +26,5 @@ data: location /media/ { alias /media/; } - # pass requests for dynamic content to gunicorn - location / { - proxy_set_header Host $host; - proxy_pass http://localhost:8080; - } } } diff --git a/docs/install/k8s/15-secrets.yaml b/docs/install/k8s/15-secrets.yaml new file mode 100644 index 00000000..33d07bb2 --- /dev/null +++ b/docs/install/k8s/15-secrets.yaml @@ -0,0 +1,10 @@ +kind: Secret +apiVersion: v1 +metadata: + name: recipes + namespace: default +type: Opaque +data: + postgresql-password: ZGItcGFzc3dvcmQ= + postgresql-postgres-password: cG9zdGdyZXMtdXNlci1wYXNzd29yZA== + secret-key: ODVkYmUxNWQ3NWVmOTMwOGM3YWUwZjMzYzdhMzI0Y2M2ZjRiZjUxOWEyZWQyZjMwMjdiZDMzYzE0MGE0ZjlhYQ== diff --git a/docs/install/k8s/20-service-account.yml b/docs/install/k8s/20-service-account.yml new file mode 100644 index 00000000..c4db988d --- /dev/null +++ b/docs/install/k8s/20-service-account.yml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: recipes + namespace: default diff --git a/docs/install/k8s/30-pv.yaml b/docs/install/k8s/30-pv.yaml deleted file mode 100644 index 810a5f34..00000000 --- a/docs/install/k8s/30-pv.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: v1 -kind: PersistentVolume -metadata: - name: recipes-db - labels: - app: recipes - type: local - tier: db -spec: - storageClassName: manual - capacity: - storage: 1Gi - accessModes: - - ReadWriteMany - hostPath: - path: "/data/recipes/db" ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: recipes-media - labels: - app: recipes - type: local - tier: media -spec: - storageClassName: manual - capacity: - storage: 1Gi - accessModes: - - ReadWriteMany - hostPath: - path: "/data/recipes/media" ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: recipes-static - labels: - app: recipes - type: local - tier: static -spec: - storageClassName: manual - capacity: - storage: 1Gi - accessModes: - - ReadWriteMany - hostPath: - path: "/data/recipes/static" diff --git a/docs/install/k8s/30-pvc.yaml b/docs/install/k8s/30-pvc.yaml index 16b8b48a..c28d19f1 100644 --- a/docs/install/k8s/30-pvc.yaml +++ b/docs/install/k8s/30-pvc.yaml @@ -1,34 +1,13 @@ apiVersion: v1 kind: PersistentVolumeClaim -metadata: - name: recipes-db - labels: - app: recipes -spec: - selector: - matchLabels: - tier: db - storageClassName: manual - accessModes: - - ReadWriteMany - resources: - requests: - storage: 1Gi ---- -apiVersion: v1 -kind: PersistentVolumeClaim metadata: name: recipes-media + namespace: default labels: app: recipes spec: - selector: - matchLabels: - tier: media - app: recipes - storageClassName: manual accessModes: - - ReadWriteMany + - ReadWriteOnce resources: requests: storage: 1Gi @@ -37,16 +16,12 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: recipes-static + namespace: default labels: app: recipes spec: - selector: - matchLabels: - tier: static - app: recipes - storageClassName: manual accessModes: - - ReadWriteMany + - ReadWriteOnce resources: requests: storage: 1Gi diff --git a/docs/install/k8s/40-sts-postgresql.yaml b/docs/install/k8s/40-sts-postgresql.yaml new file mode 100644 index 00000000..5c769dd1 --- /dev/null +++ b/docs/install/k8s/40-sts-postgresql.yaml @@ -0,0 +1,142 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app: recipes + tier: database + name: recipes-postgresql + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + app: recipes + serviceName: recipes-postgresql + updateStrategy: + type: RollingUpdate + template: + metadata: + annotations: + backup.velero.io/backup-volumes: data + labels: + app: recipes + tier: database + name: recipes-postgresql + namespace: default + spec: + restartPolicy: Always + securityContext: + fsGroup: 999 + serviceAccount: recipes + serviceAccountName: recipes + terminationGracePeriodSeconds: 30 + containers: + - name: recipes-db + env: + - name: BITNAMI_DEBUG + value: "false" + - name: POSTGRESQL_PORT_NUMBER + value: "5432" + - name: POSTGRESQL_VOLUME_DIR + value: /bitnami/postgresql + - name: PGDATA + value: /bitnami/postgresql/data + - name: POSTGRES_USER + value: recipes + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: recipes + key: postgresql-password + - name: POSTGRESQL_POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: recipes + key: postgresql-postgres-password + - name: POSTGRES_DB + value: recipes + image: docker.io/bitnami/postgresql:11.5.0-debian-9-r60 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - sh + - -c + - exec pg_isready -U "postgres" -d "wiki" -h 127.0.0.1 -p 5432 + failureThreshold: 6 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + ports: + - containerPort: 5432 + name: postgresql + protocol: TCP + readinessProbe: + exec: + command: + - sh + - -c + - -e + - | + pg_isready -U "postgres" -d "wiki" -h 127.0.0.1 -p 5432 + [ -f /opt/bitnami/postgresql/tmp/.initialized ] + failureThreshold: 6 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + resources: + requests: + cpu: 250m + memory: 256Mi + securityContext: + runAsUser: 1001 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /bitnami/postgresql + name: data + dnsPolicy: ClusterFirst + initContainers: + - command: + - sh + - -c + - | + mkdir -p /bitnami/postgresql/data + chmod 700 /bitnami/postgresql/data + find /bitnami/postgresql -mindepth 0 -maxdepth 1 -not -name ".snapshot" -not -name "lost+found" | \ + xargs chown -R 1001:1001 + image: docker.io/bitnami/minideb:stretch + imagePullPolicy: Always + name: init-chmod-data + resources: + requests: + cpu: 250m + memory: 256Mi + securityContext: + runAsUser: 0 + volumeMounts: + - mountPath: /bitnami/postgresql + name: data + restartPolicy: Always + securityContext: + fsGroup: 1001 + serviceAccount: recipes + serviceAccountName: recipes + terminationGracePeriodSeconds: 30 + updateStrategy: + type: RollingUpdate + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data + namespace: default + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + volumeMode: Filesystem diff --git a/docs/install/k8s/45-service-db.yaml b/docs/install/k8s/45-service-db.yaml new file mode 100644 index 00000000..5741d2bf --- /dev/null +++ b/docs/install/k8s/45-service-db.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: recipes + tier: database + name: recipes-postgresql + namespace: default +spec: + ports: + - name: postgresql + port: 5432 + protocol: TCP + targetPort: postgresql + selector: + app: recipes + tier: database + sessionAffinity: None + type: ClusterIP diff --git a/docs/install/k8s/50-deployment.yaml b/docs/install/k8s/50-deployment.yaml index a64e5e44..7bb6232c 100644 --- a/docs/install/k8s/50-deployment.yaml +++ b/docs/install/k8s/50-deployment.yaml @@ -2,6 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: recipes + namespace: default labels: app: recipes environment: production @@ -9,17 +10,78 @@ metadata: spec: replicas: 1 strategy: - type: RollingUpdate + type: Recreate selector: matchLabels: app: recipes environment: production template: metadata: + annotations: + backup.velero.io/backup-volumes: media,static labels: app: recipes + tier: frontend environment: production spec: + restartPolicy: Always + serviceAccount: recipes + serviceAccountName: recipes + initContainers: + - name: init-chmod-data + env: + - name: SECRET_KEY + valueFrom: + secretKeyRef: + name: recipes + key: secret-key + - name: DB_ENGINE + value: django.db.backends.postgresql_psycopg2 + - name: POSTGRES_HOST + value: recipes-postgresql + - name: POSTGRES_PORT + value: "5432" + - name: POSTGRES_USER + value: postgres + - name: POSTGRES_DB + value: recipes + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: recipes + key: postgresql-postgres-password + image: vabene1111/recipes:1.0.1 + imagePullPolicy: Always + resources: + requests: + cpu: 250m + memory: 64Mi + command: + - sh + - -c + - | + set -e + source venv/bin/activate + echo "Updating database" + python manage.py migrate + python manage.py collectstatic_js_reverse + python manage.py collectstatic --noinput + echo "Setting media file attributes" + chown -R 65534:65534 /opt/recipes/mediafiles + find /opt/recipes/mediafiles -type d | xargs -r chmod 755 + find /opt/recipes/mediafiles -type f | xargs -r chmod 644 + echo "Done" + securityContext: + runAsUser: 0 + volumeMounts: + - mountPath: /opt/recipes/mediafiles + name: media + # mount as subPath due to lost+found on ext4 pvc + subPath: files + - mountPath: /opt/recipes/staticfiles + name: static + # mount as subPath due to lost+found on ext4 pvc + subPath: files containers: - name: recipes-nginx image: nginx:latest @@ -28,69 +90,94 @@ spec: - containerPort: 80 protocol: TCP name: http + - containerPort: 8080 + protocol: TCP + name: gunicorn + resources: + requests: + cpu: 250m + memory: 64Mi volumeMounts: - - mountPath: '/media' + - mountPath: /media name: media - - mountPath: '/static' + # mount as subPath due to lost+found on ext4 pvc + subPath: files + - mountPath: /static name: static + # mount as subPath due to lost+found on ext4 pvc + subPath: files - name: nginx-config mountPath: /etc/nginx/nginx.conf subPath: nginx-config readOnly: true - name: recipes - image: 'vabene1111/recipes:latest' + image: vabene1111/recipes:1.0.1 imagePullPolicy: IfNotPresent + command: + - /opt/recipes/venv/bin/gunicorn + - -b + - :8080 + - --access-logfile + - "-" + - --error-logfile + - "-" + - --log-level + - INFO + - recipes.wsgi livenessProbe: + failureThreshold: 3 httpGet: path: / port: 8080 + scheme: HTTP + periodSeconds: 30 readinessProbe: httpGet: path: / port: 8080 + scheme: HTTP + periodSeconds: 30 + resources: + requests: + cpu: 250m + memory: 64Mi volumeMounts: - - mountPath: '/opt/recipes/mediafiles' + - mountPath: /opt/recipes/mediafiles name: media - - mountPath: '/opt/recipes/staticfiles' + # mount as subPath due to lost+found on ext4 pvc + subPath: files + - mountPath: /opt/recipes/staticfiles name: static + # mount as subPath due to lost+found on ext4 pvc + subPath: files env: - name: DEBUG value: "0" - name: ALLOWED_HOSTS value: '*' - name: SECRET_KEY - value: # CHANGEME + valueFrom: + secretKeyRef: + name: recipes + key: secret-key - name: DB_ENGINE value: django.db.backends.postgresql_psycopg2 - name: POSTGRES_HOST - value: localhost + value: recipes-postgresql - name: POSTGRES_PORT value: "5432" - name: POSTGRES_USER - value: recipes + value: postgres - name: POSTGRES_DB value: recipes - name: POSTGRES_PASSWORD - value: # CHANGEME - - name: recipes-db - image: 'postgres:latest' - imagePullPolicy: IfNotPresent - ports: - - containerPort: 5432 - volumeMounts: - - mountPath: '/var/lib/postgresql/data' - name: database - env: - - name: POSTGRES_USER - value: recipes - - name: POSTGRES_DB - value: recipes - - name: POSTGRES_PASSWORD - value: # CHANGEME + valueFrom: + secretKeyRef: + name: recipes + key: postgresql-postgres-password + securityContext: + runAsUser: 65534 volumes: - - name: database - persistentVolumeClaim: - claimName: recipes-db - name: media persistentVolumeClaim: claimName: recipes-media diff --git a/docs/install/k8s/60-service.yaml b/docs/install/k8s/60-service.yaml index 0becd59f..5a8bd61a 100644 --- a/docs/install/k8s/60-service.yaml +++ b/docs/install/k8s/60-service.yaml @@ -2,14 +2,21 @@ apiVersion: v1 kind: Service metadata: name: recipes + namespace: default labels: app: recipes + tier: frontend spec: selector: app: recipes + tier: frontend environment: production ports: - port: 80 targetPort: http name: http protocol: TCP + - port: 8080 + targetPort: gunicorn + name: gunicorn + protocol: TCP diff --git a/docs/install/k8s/70-ingress.yaml b/docs/install/k8s/70-ingress.yaml new file mode 100644 index 00000000..01bdeebd --- /dev/null +++ b/docs/install/k8s/70-ingress.yaml @@ -0,0 +1,38 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + #cert-manager.io/cluster-issuer: letsencrypt-prod + #kubernetes.io/ingress.class: nginx + name: recipes + namespace: default +spec: + rules: + - host: recipes.local + http: + paths: + - backend: + service: + name: recipes + port: + number: 8080 + path: / + pathType: Prefix + - backend: + service: + name: recipes + port: + number: 80 + path: /media + pathType: Prefix + - backend: + service: + name: recipes + port: + number: 80 + path: /static + pathType: Prefix + #tls: + #- hosts: + # - recipes.local + # secretName: recipes-local-tls diff --git a/docs/install/kubernetes.md b/docs/install/kubernetes.md index 4f29f97f..992be89c 100644 --- a/docs/install/kubernetes.md +++ b/docs/install/kubernetes.md @@ -1,31 +1,98 @@ -!!! info "Community Contributed" - This guide was contributed by the community and is neither officially supported, nor updated or tested. +**!!! info "Community Contributed" This guide was contributed by the community and is neither officially supported, nor updated or tested.** -This is a basic kubernetes setup. -Please note that this does not necessarily follow Kubernetes best practices and should only used as a -basis to build your own setup from! +# K8s Setup -All files con be found here in the Github Repo: -[docs/install/k8s](https://github.com/vabene1111/recipes/tree/develop/docs/install/k8s) +This is a setup which should be sufficent for production use. Be sure to replace the default secrets! -## Important notes +# Files -State (database, static files and media files) is handled via `PersistentVolumes`. +## 10-configmap.yaml -Note that you will most likely have to change the `PersistentVolumes` in `30-pv.yaml`. The current setup is only usable for a single-node cluster because it uses local storage on the kubernetes worker nodes under `/data/recipes/`. It should just serve as an example. +The nginx config map. This is loaded as nginx.conf in the nginx sidecar to configure nginx to deliver static content. -Currently, the deployment in `50-deployment.yaml` just pulls the `latest` tag of all containers. In a production setup, you should set this to a fixed version! +## 15-secrets.yaml -See env variables tagged with `CHANGEME` in `50-deployment.yaml` and make sure to change those! A better setup would use kubernetes secrets but this is not implemented yet. +The secrets **replace them!!** This file is only here for a quick start. Be aware that changing secrets after installation will be messy and is not documented here. **You should set new secrets before the installation.** As you are reading this document **before** the installation ;-) -## Updates +Create your own postgresql passwords and the secret key for the django app -These manifests are not tested against new versions. +see also [Managing Secrets using kubectl](https://kubernetes.io/docs/tasks/configmap-secret/managing-secret-using-kubectl/) -## Apply the manifets +**Replace** `db-password`, `postgres-user-password` and `secret-key` **with something - well - secret :-)** -To apply the manifest with `kubectl`, use the following command: +~~~ +echo -n 'db-password' > ./db-password.txt +echo -n 'postgres-user-password' > ./postgres-password.txt +echo -n 'secret-key' | sha256sum | awk '{ printf $1 }' > ./secret-key.txt +~~~ -``` +Delete the default secrets file `15-secrets.yaml` and generate the K8s secret from your files. + +~~~ +kubectl create secret generic recipes \ + --from-file=postgresql-password=./db-password.txt \ + --from-file=postgresql-postgres-password=./postgres-password.txt \ + --from-file=secret-key=./secret-key.txt +~~~ + +## 20-service-account.yml + +Creating service account `recipes` for deployment and stateful set. + +## 30-pvc.yaml + +The creation of the persistent volume claims for media and static content. May you want to increase the size. This expects to have a storage class installed. + +## 40-sts-postgresql.yaml + +The PostgreSQL stateful set, based on a bitnami image. It runs a init container as root to do the preparations. The postgres container itsef runs as a lower privileged user. The recipes app uses the database super user (postgres) as the recipies app is doing some db migrations on startup, which needs super user privileges. + +## 45-service-db.yaml + +Creating the database service. + +## 50-deployment.yaml + +The deployment first fires up a init container to do the database migrations and file modifications. This init container runs as root. The init conainer runs part of the [boot.sh](https://github.com/TandoorRecipes/recipes/blob/develop/boot.sh) script from the `vabene1111/recipes` image. + +The deployment then runs two containers, the recipes-nginx and the recipes container which runs the gunicorn app. The nginx container gets it's nginx.conf via config map to deliver static content `/static` and `/media`. The guincorn container gets it's secret key and the database password from the secret `recipes`. `gunicorn` runs as user `nobody`. + +## 60-service.yaml + +Creating the app service. + +## 70-ingress.yaml + +Setting up the ingress for the recipes service. Requests for static content `/static` and `/media` are send to the nginx container, everything else to gunicorn. TLS setup via cert-manager is prepared. You have to **change the host** from `recipes.local` to your specific domain. + +# Conclusion + +All in all: + +- The database is set up as a stateful set. +- The database container runs as a low privileged user. +- Database and application use secrets. +- The application also runs as a low privileged user. +- nginx runs as root but forks children with a low privileged user. +- There's an ingress rule to access the application from outside. + +I tried the setup with [kind](https://kind.sigs.k8s.io/) and it runs well on my local cluster. + +There is a warning, when you check your system as super user: + +**Media Serving Warning** +Serving media files directly using gunicorn/python is not recommend! Please follow the steps described here to update your installation. + +I don't know how this check works, but this warning is simply wrong! ;-) Media and static files are routed by ingress to the nginx container - I promise :-) + +# Updates + +These manifests are tested against Release 1.0.1. Newer versions may not work without changes. + +# Apply the manifets + +To apply the manifest with kubectl, use the following command: + +~~~ kubectl apply -f ./docs/k8s/ -``` +~~~