From bed5ed5767132cd6cf385f032c7043c8450b0cf7 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Sat, 8 Feb 2025 10:44:19 -0600 Subject: [PATCH] dch-webhooks: Enable host provisioning feature The *dch-webhooks* server now has a _POST /host/online_ hook that can be triggered by a new machine when it first comes online. This hook starts an automatic provisioning process by creating a Kubernetes Job to run Ansible and publishing information about the host to provision via AMQP. Thus, the server now needs access to the Kubernetes API in order to create the Job and access to RabbitMQ in order to publish the task parameters. --- ansible/.gitignore | 1 + ansible/host-provisioner.key.pub | 1 + ansible/kustomization.yaml | 2 + ansible/rbac.yaml | 25 +++++++ ansible/secrets.yaml | 18 +++++ dch-webhooks/ansible-job.yaml | 115 +++++++++++++++++++++++++++++++ dch-webhooks/certificate.yaml | 14 ++++ dch-webhooks/dch-webhooks.env | 7 ++ dch-webhooks/dch-webhooks.yaml | 29 +++++++- dch-webhooks/kustomization.yaml | 14 ++++ 10 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 ansible/host-provisioner.key.pub create mode 100644 ansible/rbac.yaml create mode 100644 dch-webhooks/ansible-job.yaml create mode 100644 dch-webhooks/certificate.yaml diff --git a/ansible/.gitignore b/ansible/.gitignore index 2920ec7..667f6cf 100644 --- a/ansible/.gitignore +++ b/ansible/.gitignore @@ -1 +1,2 @@ ara/.secrets.toml +host-provisioner.key diff --git a/ansible/host-provisioner.key.pub b/ansible/host-provisioner.key.pub new file mode 100644 index 0000000..04db65e --- /dev/null +++ b/ansible/host-provisioner.key.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICoOO/ZYMxRgmyvqZwGN3NM5pHyh3NBdC7iZrXIopt93 Host Provisioner diff --git a/ansible/kustomization.yaml b/ansible/kustomization.yaml index d2317b3..3d21bc9 100644 --- a/ansible/kustomization.yaml +++ b/ansible/kustomization.yaml @@ -13,6 +13,8 @@ namespace: ansible resources: - ../dch-root-ca +- ../ssh-host-keys +- rbac.yaml - secrets.yaml - namespace.yaml - ara.yaml diff --git a/ansible/rbac.yaml b/ansible/rbac.yaml new file mode 100644 index 0000000..4067edf --- /dev/null +++ b/ansible/rbac.yaml @@ -0,0 +1,25 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: dch-webhooks +rules: + - apiGroups: + - batch + resources: + - jobs + verbs: + - create + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: dch-webhooks +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: dch-webhooks +subjects: +- kind: ServiceAccount + name: dch-webhooks + namespace: default diff --git a/ansible/secrets.yaml b/ansible/secrets.yaml index c75eec1..7422b26 100644 --- a/ansible/secrets.yaml +++ b/ansible/secrets.yaml @@ -17,3 +17,21 @@ spec: labels: app.kubernetes.io/name: ara app.kubernetes.io/component: ara + +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: provisioner-ssh-key + namespace: ansible + labels: &labels + app.kubernetes.io/name: provisioner-ssh-key + app.kubernetes.io/component: host-provisioner +spec: + encryptedData: + host-provisioner.key: AgCiBQEtPmzQO9GHoVxZNQmjFn9GjeKWlHmG2lz4uqeva77QByqnejUg8CbpQnoT61ct9o39tVtrrHgC8WjseXKGKkd+YELRaWENXBNBFlv4YN6id2iPzf4xHrgfowXLjzly7s4s4Fg3QntXFglQxwG009Z7K5HQmAvgCFzGm6y+fyLoNd2v2eNc9pGurynnm7wKQFuBBpBtoDVfYMNSS+tp9D3MFVMwEG8kU/kszv5OEEwVkipG4v2LeY86HXIQnHyeE3vITz0TFPoQjpjusBNg7MxBVHcz/ZruqF43DZ+uz2aRFL6D5udm86hsxZGSi3yVq8PSCUANtWoTicIpSc5yxpB+5FLsc3Q380vQPrRYvx8luAZwMGQPDv2NpGoSRUmutb/+4vpQbCBwaP9s8WHaYNDByUeoQKAX2o+RYp+/csfxmSy1/pK93RUjMnsWGYiyI7jpNdTqWkakDN+cM0Lrje9+VqcZge3FYp/4y78AI75pWEiEMy1VSXeqE2pgu5vZzyRdw9zORC2sAWhnTeu+Obbly1UdptpXPmGclmRbpwAPM1F7m7pDsCQ9SYAhoGg251Yu0sF3Nnc0uOElmP0KSn1bt/Jca9M0syg9DMntHo41dn40+Aihujej0ll4h3GXVZ0auUuzSLZEMe0dHQY0YCaq3JdeOa6AJuNyo63/0BmvmWP2T06INtVw4EqBsmvNl/YJIlZiRGhIOGaRyJllu21L6QBtToYjxI7fhTxZaCG6fyPbvCd1hu+IchoMr86rl7W0+k6d/lkamx9dg+PtWjjThOGyeLYwSRhMcsvQPsSdTl8BEmB9TRgRIk42txVRQkb+ei1+1y9nBYCOV9P540lDXQVbqgyb0j2cccWOa/AG7l3P3s8haDs2OujUaVkBJWJ489nsLsC33972wwAQbOk+uTKmXKL6nWSNL31rivW9R8ea+vfdcyN2/tLsI0mE9XTR/kRmS12qdLmDbzeX/scqSKaWFQZwnLHak5Xaqkd25ZGHqB3zrnTBIryDaXSzgC1CPCv7QNqbKvd6vmH+16j1DWqLycbUeorx5O9nwpEZ+GzQm8q62PJEO8xdvqr3nu5OvFgGrnX/Y6giV8Cv8ogyAxbmNrq0cC5VYxDlt4VvzlOpmWUlfxD/wd+gaHQoibhhHRbGtsJCX9AFclVQrY3kmduTi/iJlTTJjMQKm9JJRXKbNDxH+B25Zj/hUIC1/NNZOSDn654Pi/yOGXITp86j9unfnIXdJPg= + template: + metadata: + name: provisioner-ssh-key + namespace: ansible + labels: *labels diff --git a/dch-webhooks/ansible-job.yaml b/dch-webhooks/ansible-job.yaml new file mode 100644 index 0000000..f8d5c43 --- /dev/null +++ b/dch-webhooks/ansible-job.yaml @@ -0,0 +1,115 @@ +apiVersion: batch/v1 +kind: Job +metadata: + generateName: host-provision- + labels: &labels + app.kubernetes.io/name: host-provisioner + app.kubernetes.io/component: host-provisioner +spec: + backoffLimit: 0 + template: + metadata: + labels: *labels + spec: + restartPolicy: Never + initContainers: + - name: ssh-agent + image: &image git.pyrocufflink.net/infra/host-provisioner + imagePullPolicy: Always + command: + - tini + - ssh-agent + - -- + - -D + - -a + - /run/ssh/agent.sock + restartPolicy: Always + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /run/ssh + name: tmp + subPath: run/ssh + - name: ssh-add + image: *image + command: + - ssh-add + - -t + - 30m + - /run/secrets/ssh/host-provisioner.key + env: + - name: SSH_AUTH_SOCK + value: /run/ssh/agent.sock + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /run/ssh + name: tmp + subPath: run/ssh + - mountPath: /run/secrets/ssh + name: provisioner-key + readOnly: true + containers: + - name: host-provisioner + image: *image + env: + - name: SSH_AUTH_SOCK + value: /run/ssh/agent.sock + - name: AMQP_HOST + value: rabbitmq.pyrocufflink.blue + - name: AMQP_PORT + value: '5671' + - name: AMQP_CA_CERT + value: /run/dch-ca/dch-root-ca.crt + - name: AMQP_CLIENT_CERT + value: /run/secrets/host-provisioner/rabbitmq/tls.crt + - name: AMQP_CLIENT_KEY + value: /run/secrets/host-provisioner/rabbitmq/tls.key + - name: AMQP_EXTERNAL_CREDENTIALS + value: '1' + securityContext: + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /etc/ssh/ssh_known_hosts + name: ssh-known-hosts + subPath: ssh_known_hosts + readOnly: true + - mountPath: /home/jenkins + name: workspace + - mountPath: /run/dch-ca + name: dch-root-ca + readOnly: true + - mountPath: /run/ssh + name: tmp + subPath: run/ssh + - mountPath: /run/secrets/host-provisioner/rabbitmq + name: rabbitmq-cert + readOnly: true + - mountPath: /tmp + name: tmp + subPath: tmp + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + volumes: + - name: dch-root-ca + configMap: + name: dch-root-ca + - name: provisioner-key + secret: + secretName: provisioner-ssh-key + defaultMode: 0440 + - name: ssh-known-hosts + configMap: + name: ssh-known-hosts + - name: rabbitmq-cert + secret: + secretName: rabbitmq-cert + defaultMode: 0440 + - name: tmp + emptyDir: + medium: Memory + - name: workspace + emptyDir: {} diff --git a/dch-webhooks/certificate.yaml b/dch-webhooks/certificate.yaml new file mode 100644 index 0000000..af1e7ba --- /dev/null +++ b/dch-webhooks/certificate.yaml @@ -0,0 +1,14 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: rabbitmq +spec: + secretName: rabbitmq-cert + commonName: dch-webhooks + issuerRef: + group: cert-manager.io + kind: ClusterIssuer + name: rabbitmq-ca + privateKey: + algorithm: ECDSA + rotationPolicy: Always diff --git a/dch-webhooks/dch-webhooks.env b/dch-webhooks/dch-webhooks.env index fb06c57..e8ae63c 100644 --- a/dch-webhooks/dch-webhooks.env +++ b/dch-webhooks/dch-webhooks.env @@ -7,3 +7,10 @@ STEP_CA_URL=https://ca.pyrocufflink.blue:32599 STEP_ROOT=/run/dch-root-ca.crt STEP_PROVISIONER=host-bootstrap STEP_PROVISIONER_PASSWORD_FILE=/run/secrets/du5t1n.me/step-ca/provisioner.password + +AMQP_HOST=rabbitmq.pyrocufflink.blue +AMQP_PORT=5671 +AMQP_EXTERNAL_CREDENTIALS=1 +AMQP_CA_CERT=/run/dch-root-ca.crt +AMQP_CLIENT_CERT=/run/secrets/du5t1n.me/rabbitmq/tls.crt +AMQP_CLIENT_KEY=/run/secrets/du5t1n.me/rabbitmq/tls.key diff --git a/dch-webhooks/dch-webhooks.yaml b/dch-webhooks/dch-webhooks.yaml index 5a6f909..704bf87 100644 --- a/dch-webhooks/dch-webhooks.yaml +++ b/dch-webhooks/dch-webhooks.yaml @@ -1,4 +1,14 @@ apiVersion: v1 +kind: ServiceAccount +metadata: + name: dch-webhooks + labels: + app.kubernetes.io/name: dch-webhooks + app.kubernetes.io/component: dch-webhooks + app.kubernetes.io/part-of: dch-webhooks + +--- +apiVersion: v1 kind: Service metadata: labels: @@ -42,12 +52,14 @@ spec: spec: containers: - name: dch-webhooks - image: git.pyrocufflink.net/containerimages/dch-webhooks + image: git.pyrocufflink.net/infra/dch-webhooks env: - name: UVICORN_HOST value: 0.0.0.0 - name: UVICORN_LOG_LEVEL value: debug + - name: ANSIBLE_JOB_YAML + value: /etc/dch-webhooks/ansible-job.yaml envFrom: - configMapRef: name: dch-webhooks @@ -76,22 +88,37 @@ spec: name: firefly-token - mountPath: /run/secrets/du5t1n.me/paperless name: paperless-token + - mountPath: /run/secrets/du5t1n.me/rabbitmq + name: rabbitmq-cert + readOnly: true - mountPath: /run/secrets/du5t1n.me/step-ca name: step-ca-password - mountPath: /tmp name: tmp subPath: tmp + - mountPath: /etc/dch-webhooks + name: host-provisioner + readOnly: true securityContext: runAsNonRoot: true + serviceAccountName: dch-webhooks volumes: - name: firefly-token secret: secretName: firefly-token optional: true + - name: host-provisioner + configMap: + name: host-provisioner + optional: true - name: paperless-token secret: secretName: paperless-token optional: true + - name: rabbitmq-cert + secret: + secretName: rabbitmq-cert + optional: true - name: root-ca configMap: name: dch-root-ca diff --git a/dch-webhooks/kustomization.yaml b/dch-webhooks/kustomization.yaml index d3a395e..d791cef 100644 --- a/dch-webhooks/kustomization.yaml +++ b/dch-webhooks/kustomization.yaml @@ -1,15 +1,29 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +labels: +- pairs: + app.kubernetes.io/instance: dch-webhooks + includeSelectors: true + includeTemplates: true +- pairs: + app.kubernetes.io/part-of: dch-webhooks + resources: - ../dch-root-ca - dch-webhooks.yaml +- certificate.yaml - ingress.yaml configMapGenerator: - name: dch-webhooks envs: - dch-webhooks.env +- name: host-provisioner + files: + - ansible-job.yaml + options: + disableNameSuffixHash: true secretGenerator: - name: firefly-token