diff --git a/.editorconfig b/.editorconfig index 93f4e5f..fb078e0 100644 --- a/.editorconfig +++ b/.editorconfig @@ -13,3 +13,6 @@ max_line_length = 79 max_line_length = 79 indent_style = space indent_size = 4 + +[**.tf] +indent_size = 2 diff --git a/terraform/asg.tf b/terraform/asg.tf new file mode 100644 index 0000000..4b6baf0 --- /dev/null +++ b/terraform/asg.tf @@ -0,0 +1,71 @@ +resource "aws_security_group" "k8s-node" { + name = "k8s-node" + description = "Kubernetes Node" + + egress { + from_port = 19998 + to_port = 19998 + protocol = "udp" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + + egress { + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } + + egress { + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + ipv6_cidr_blocks = ["::/0"] + } +} + +resource "aws_launch_template" "k8s-aarch64" { + name = "k8s-aarch64" + + update_default_version = true + image_id = "ami-000ec96ccb51eb679" + instance_type = "t4g.medium" + security_group_names = [aws_security_group.k8s-node.name] + + user_data = filebase64("${path.module}/userdata.yml") + + instance_market_options { + market_type = "spot" + } + + private_dns_name_options { + hostname_type = "resource-name" + } +} + +resource "aws_autoscaling_group" "k8s-aarch64" { + name = "k8s-aarch64" + + availability_zones = ["us-east-2a", "us-east-2b", "us-east-2c"] + min_size = 0 + max_size = 1 + + launch_template { + id = aws_launch_template.k8s-aarch64.id + version = "$Latest" + } + + tag { + key = "k8s.io/cluster-autoscaler/enabled" + value = "true" + propagate_at_launch = true + } + tag { + key = "k8s.io/cluster-autoscaler/kubernetes" + value = "owned" + propagate_at_launch = true + } +} diff --git a/terraform/iam-policy.json b/terraform/iam-policy.json index 784722b..3ed129b 100644 --- a/terraform/iam-policy.json +++ b/terraform/iam-policy.json @@ -5,45 +5,75 @@ "Sid": "VisualEditor0", "Effect": "Allow", "Action": [ - "sns:ListTagsForResource", - "events:DescribeRule", - "sns:GetTopicAttributes", - "events:EnableRule", - "sns:DeleteTopic", - "events:PutRule", - "sns:CreateTopic", - "sns:SetTopicAttributes", - "events:DeleteRule", - "events:PutTargets", - "events:ListTagsForResource", - "sns:Subscribe", - "events:RemoveTargets", - "events:ListTargetsByRule", - "events:DisableRule" + "ec2:DescribeLaunchTemplates", + "autoscaling:DescribeAutoScalingGroups", + "ec2:DescribeLaunchTemplateVersions", + "autoscaling:DescribeTags", + "sns:Unsubscribe", + "sns:GetSubscriptionAttributes", + "ec2:DescribeSecurityGroups" ], - "Resource": [ - "arn:aws:sns:*:566967686773:*", - "arn:aws:events:*:566967686773:rule/*" - ] + "Resource": "*" }, { "Sid": "VisualEditor1", "Effect": "Allow", "Action": [ - "events:DescribeEventBus", + "autoscaling:DeleteTags", + "sns:ListTagsForResource", + "ec2:UpdateSecurityGroupRuleDescriptionsEgress", + "sns:GetTopicAttributes", + "ec2:DeleteTags", "events:CreateEventBus", - "events:DeleteEventBus" + "sns:DeleteTopic", + "ec2:CreateTags", + "sns:CreateTopic", + "sns:SetTopicAttributes", + "ec2:ModifySecurityGroupRules", + "ec2:UpdateSecurityGroupRuleDescriptionsIngress", + "events:DescribeEventBus", + "ec2:RevokeSecurityGroupIngress", + "autoscaling:CreateOrUpdateTags", + "ec2:CreateSecurityGroup", + "ec2:RevokeSecurityGroupEgress", + "ec2:DeleteSecurityGroup", + "events:DeleteEventBus", + "autoscaling:UpdateAutoScalingGroup", + "sns:Subscribe", + "autoscaling:DeleteAutoScalingGroup", + "autoscaling:CreateAutoScalingGroup" ], - "Resource": "arn:aws:events:*:566967686773:event-bus/*" + "Resource": [ + "arn:aws:events:*:566967686773:event-bus/*", + "arn:aws:autoscaling:*:566967686773:autoScalingGroup:*:autoScalingGroupName/*", + "arn:aws:sns:*:566967686773:*", + "arn:aws:ec2:*:566967686773:security-group/*", + "arn:aws:ec2:*:566967686773:security-group-rule/*" + ] }, { "Sid": "VisualEditor2", "Effect": "Allow", "Action": [ - "sns:Unsubscribe", - "sns:GetSubscriptionAttributes" + "events:DescribeRule", + "ec2:DeleteLaunchTemplate", + "events:EnableRule", + "events:PutRule", + "ec2:CreateLaunchTemplateVersion", + "events:DeleteRule", + "events:PutTargets", + "ec2:CreateLaunchTemplate", + "events:ListTagsForResource", + "events:RemoveTargets", + "ec2:ModifyLaunchTemplate", + "ec2:DeleteLaunchTemplateVersions", + "events:ListTargetsByRule", + "events:DisableRule" ], - "Resource": "*" + "Resource": [ + "arn:aws:events:*:566967686773:rule/*", + "arn:aws:ec2:*:566967686773:launch-template/*" + ] } ] } diff --git a/terraform/terraform.tfstate b/terraform/terraform.tfstate index b27fdbe..e59c274 100644 --- a/terraform/terraform.tfstate +++ b/terraform/terraform.tfstate @@ -1,7 +1,7 @@ { "version": 4, "terraform_version": "1.2.9", - "serial": 49, + "serial": 78, "lineage": "a100be74-c98e-0769-2d6a-bf6a2c5f3ebf", "outputs": {}, "resources": [ @@ -15,9 +15,9 @@ "schema_version": 0, "attributes": { "account_id": "566967686773", - "arn": "arn:aws:sts::566967686773:assumed-role/dynk8s-terraform/aws-go-sdk-1664301518318294107", + "arn": "arn:aws:sts::566967686773:assumed-role/dynk8s-terraform/aws-go-sdk-1665542385873038019", "id": "566967686773", - "user_id": "AROAYIAPIKZ25DFDOYZHT:aws-go-sdk-1664301518318294107" + "user_id": "AROAYIAPIKZ25DFDOYZHT:aws-go-sdk-1665542385873038019" }, "sensitive_attributes": [] } @@ -107,6 +107,85 @@ } ] }, + { + "mode": "managed", + "type": "aws_autoscaling_group", + "name": "k8s-aarch64", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:autoscaling:us-east-2:566967686773:autoScalingGroup:e5c074b6-9c58-478c-92ec-f0685465ca3d:autoScalingGroupName/k8s-aarch64", + "availability_zones": [ + "us-east-2a", + "us-east-2b", + "us-east-2c" + ], + "capacity_rebalance": false, + "context": "", + "default_cooldown": 300, + "default_instance_warmup": 0, + "desired_capacity": 0, + "enabled_metrics": [], + "force_delete": false, + "force_delete_warm_pool": false, + "health_check_grace_period": 300, + "health_check_type": "EC2", + "id": "k8s-aarch64", + "initial_lifecycle_hook": [], + "instance_refresh": [], + "launch_configuration": "", + "launch_template": [ + { + "id": "lt-0789a3800bdaec215", + "name": "k8s-aarch64", + "version": "$Latest" + } + ], + "load_balancers": [], + "max_instance_lifetime": 0, + "max_size": 1, + "metrics_granularity": "1Minute", + "min_elb_capacity": null, + "min_size": 0, + "mixed_instances_policy": [], + "name": "k8s-aarch64", + "name_prefix": "", + "placement_group": "", + "protect_from_scale_in": false, + "service_linked_role_arn": "arn:aws:iam::566967686773:role/aws-service-role/autoscaling.amazonaws.com/AWSServiceRoleForAutoScaling", + "suspended_processes": [], + "tag": [ + { + "key": "k8s.io/cluster-autoscaler/enabled", + "propagate_at_launch": true, + "value": "true" + }, + { + "key": "k8s.io/cluster-autoscaler/kubernetes", + "propagate_at_launch": true, + "value": "owned" + } + ], + "tags": null, + "target_group_arns": [], + "termination_policies": [], + "timeouts": null, + "vpc_zone_identifier": [], + "wait_for_capacity_timeout": "10m", + "wait_for_elb_capacity": null, + "warm_pool": [] + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiZGVsZXRlIjo2MDAwMDAwMDAwMDAsInVwZGF0ZSI6NjAwMDAwMDAwMDAwfX0=", + "dependencies": [ + "aws_launch_template.k8s-aarch64", + "aws_security_group.k8s-node" + ] + } + ] + }, { "mode": "managed", "type": "aws_cloudwatch_event_rule", @@ -171,6 +250,152 @@ } ] }, + { + "mode": "managed", + "type": "aws_launch_template", + "name": "k8s-aarch64", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:566967686773:launch-template/lt-0789a3800bdaec215", + "block_device_mappings": [], + "capacity_reservation_specification": [], + "cpu_options": [], + "credit_specification": [], + "default_version": 6, + "description": "", + "disable_api_stop": false, + "disable_api_termination": false, + "ebs_optimized": "", + "elastic_gpu_specifications": [], + "elastic_inference_accelerator": [], + "enclave_options": [], + "hibernation_options": [], + "iam_instance_profile": [], + "id": "lt-0789a3800bdaec215", + "image_id": "ami-000ec96ccb51eb679", + "instance_initiated_shutdown_behavior": "", + "instance_market_options": [ + { + "market_type": "spot", + "spot_options": [] + } + ], + "instance_requirements": [], + "instance_type": "t4g.medium", + "kernel_id": "", + "key_name": "", + "latest_version": 6, + "license_specification": [], + "maintenance_options": [], + "metadata_options": [], + "monitoring": [], + "name": "k8s-aarch64", + "name_prefix": "", + "network_interfaces": [], + "placement": [], + "private_dns_name_options": [ + { + "enable_resource_name_dns_a_record": false, + "enable_resource_name_dns_aaaa_record": false, + "hostname_type": "resource-name" + } + ], + "ram_disk_id": "", + "security_group_names": [ + "k8s-node" + ], + "tag_specifications": [], + "tags": {}, + "tags_all": {}, + "update_default_version": true, + "user_data": "I2Nsb3VkLWNvbmZpZwpib290Y21kOgotIFsgZG5mLCBtb2R1bGUsIGVuYWJsZSwgJ2NyaS1vOjEuMjInLCAteSBdCi0gWyBsbiwgLXNmLCAvcnVuL3N5c3RlbWQvcmVzb2x2ZS9zdHViLXJlc29sdi5jb25mLCAvZXRjL3Jlc29sdi5jb25mIF0KCnBhY2thZ2VzOgotIGNyaS1vCi0gY3JpLXRvb2xzCi0gZXRodG9vbAotIGlwdGFibGVzLW5mdAotIGlzY3NpLWluaXRpYXRvci11dGlscwotIGt1YmVybmV0ZXMta3ViZWFkbQotIGt1YmVybmV0ZXMtbm9kZQotIHdpcmVndWFyZC10b29scwoKd3JpdGVfZmlsZXM6Ci0gcGF0aDogL2V0Yy9kbmYvZG5mLmNvbmYKICBjb250ZW50OiB8KwogICAgaW5zdGFsbF93ZWFrX2RlcHM9RmFsc2UKICBhcHBlbmQ6IHRydWUKLSBwYXRoOiAvZXRjL21vZHVsZXMtbG9hZC5kL2s4cy5jb25mCiAgY29udGVudDogfCsKICAgIGJyX25ldGZpbHRlcgotIHBhdGg6IC9ldGMvc3lzY3RsLmQvazhzLmNvbmYKICBjb250ZW50OiB8KwogICAgbmV0LmJyaWRnZS5icmlkZ2UtbmYtY2FsbC1pcHRhYmxlcyA9IDEKICAgIG5ldC5icmlkZ2UuYnJpZGdlLW5mLWNhbGwtaXA2dGFibGVzID0gMQogICAgbmV0LmlwdjQuaXBfZm9yd2FyZCA9IDEKLSBwYXRoOiAvdmFyL2xpYi9jbG91ZC9zY3JpcHRzL3Blci1pbnN0YW5jZS9rdWJlYWRtLWpvaW4KICBwZXJtaXNzaW9uczogJzA3NTUnCiAgY29udGVudDogfCsKICAgICMhL2Jpbi9zaAoKICAgIEJBU0VfVVJMPWh0dHBzOi8vZHluazhzLXByb3Zpc2lvbmVyLnB5cm9jdWZmbGluay5uZXQKCiAgICBpbnN0YW5jZV9pZD0kKGN1cmwgLXMgMTY5LjI1NC4xNjkuMjU0L2xhdGVzdC9tZXRhLWRhdGEvaW5zdGFuY2UtaWQpCiAgICBhej0kKGN1cmwgLXMgMTY5LjI1NC4xNjkuMjU0L2xhdGVzdC9tZXRhLWRhdGEvcGxhY2VtZW50L2F2YWlsYWJpbGl0eS16b25lKQoKICAgIGN1cmwgLWZzICIke0JBU0VfVVJMfSIvd2lyZWd1YXJkL2NvbmZpZy8ke2luc3RhbmNlX2lkfSBcCiAgICAgICAgLW8gL2V0Yy93aXJlZ3VhcmQvd2cwLmNvbmYgfHwgZXhpdAogICAgc3lzdGVtY3RsIGVuYWJsZSAtLW5vdyB3Zy1xdWlja0B3ZzAgfHwgZXhpdAoKICAgIG1vZHByb2JlIGJyX25ldGZpbHRlciB8fCBleGl0CiAgICBzeXNjdGwgLXcgLWYgL2V0Yy9zeXNjdGwuZC9rOHMuY29uZiB8fCBleGl0CgogICAgc3dhcG9mZiAtYSB8fCBleGl0CiAgICB0b3VjaCAvZXRjL3N5c3RlbWQvenJhbS1nZW5lcmF0b3IuY29uZiB8fCBleGl0CiAgICBzeXN0ZW1jdGwgZGFlbW9uLXJlbG9hZCB8fCBleGl0CiAgICBzeXN0ZW1jdGwgc3RvcCAnc3lzdGVtZC16cmFtLXNldHVwQConIHx8IGV4aXQKCiAgICBzeXN0ZW1jdGwgZW5hYmxlIGNyaW8gaXNjc2lkIGt1YmVsZXQgfHwgZXhpdAogICAgc3lzdGVtY3RsIHN0YXJ0IGNyaW8gaXNjc2lkIHx8IGV4aXQKCiAgICBpbnRlcm5hbF9pcD0kKAogICAgICBpcCBhZGRyZXNzIHNob3cgZGV2IHdnMCBwcmltYXJ5IHwgXAogICAgICBzZWQgLXJuICdzLy4qaW5ldCAoWzAtOS5dKykuKi9cMS9wJwogICAgKQoKICAgIGNhdCA+IC9ydW4vam9pbmNvbmZpZ3VyYXRpb24gPDxFT0YKICAgIGFwaVZlcnNpb246IGt1YmVhZG0uazhzLmlvL3YxYmV0YTMKICAgIGtpbmQ6IEpvaW5Db25maWd1cmF0aW9uCiAgICBub2RlUmVnaXN0cmF0aW9uOgogICAgICBrdWJlbGV0RXh0cmFBcmdzOgogICAgICAgIHByb3ZpZGVyLWlkOiBhd3M6Ly8vJHthen0vJHtpbnN0YW5jZV9pZH0KICAgICAgICBub2RlLWlwOiAke2ludGVybmFsX2lwfQogICAgZGlzY292ZXJ5OgogICAgICBmaWxlOgogICAgICAgIGt1YmVDb25maWdQYXRoOiAke0JBU0VfVVJMfS9rdWJlYWRtL2t1YmVjb25maWcvJHtpbnN0YW5jZV9pZH0KICAgIEVPRgogICAga3ViZWFkbSBqb2luIC0tY29uZmlnPS9ydW4vam9pbmNvbmZpZ3VyYXRpb24KCnJ1bmNtZDoKLSBbIGRuZiwgcmVtb3ZlLCAteSwgenJhbS1nZW5lcmF0b3IgXQo=", + "vpc_security_group_ids": [] + }, + "sensitive_attributes": [], + "private": "bnVsbA==", + "dependencies": [ + "aws_security_group.k8s-node" + ] + } + ] + }, + { + "mode": "managed", + "type": "aws_security_group", + "name": "k8s-node", + "provider": "provider[\"registry.terraform.io/hashicorp/aws\"]", + "instances": [ + { + "schema_version": 1, + "attributes": { + "arn": "arn:aws:ec2:us-east-2:566967686773:security-group/sg-05258c3ff1812e83b", + "description": "Kubernetes Node", + "egress": [ + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "", + "from_port": 19998, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "udp", + "security_groups": [], + "self": false, + "to_port": 19998 + }, + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "", + "from_port": 443, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 443 + }, + { + "cidr_blocks": [ + "0.0.0.0/0" + ], + "description": "", + "from_port": 80, + "ipv6_cidr_blocks": [ + "::/0" + ], + "prefix_list_ids": [], + "protocol": "tcp", + "security_groups": [], + "self": false, + "to_port": 80 + } + ], + "id": "sg-05258c3ff1812e83b", + "ingress": [], + "name": "k8s-node", + "name_prefix": "", + "owner_id": "566967686773", + "revoke_rules_on_delete": false, + "tags": {}, + "tags_all": {}, + "timeouts": null, + "vpc_id": "vpc-2483044d" + }, + "sensitive_attributes": [], + "private": "eyJlMmJmYjczMC1lY2FhLTExZTYtOGY4OC0zNDM2M2JjN2M0YzAiOnsiY3JlYXRlIjo2MDAwMDAwMDAwMDAsImRlbGV0ZSI6OTAwMDAwMDAwMDAwfSwic2NoZW1hX3ZlcnNpb24iOiIxIn0=" + } + ] + }, { "mode": "managed", "type": "aws_sns_topic", diff --git a/terraform/userdata.yml b/terraform/userdata.yml new file mode 100644 index 0000000..9772c34 --- /dev/null +++ b/terraform/userdata.yml @@ -0,0 +1,73 @@ +#cloud-config +bootcmd: +- [ dnf, module, enable, 'cri-o:1.22', -y ] +- [ ln, -sf, /run/systemd/resolve/stub-resolv.conf, /etc/resolv.conf ] + +packages: +- cri-o +- cri-tools +- ethtool +- iptables-nft +- iscsi-initiator-utils +- kubernetes-kubeadm +- kubernetes-node +- wireguard-tools + +write_files: +- path: /etc/dnf/dnf.conf + content: |+ + install_weak_deps=False + append: true +- path: /etc/modules-load.d/k8s.conf + content: |+ + br_netfilter +- path: /etc/sysctl.d/k8s.conf + content: |+ + net.bridge.bridge-nf-call-iptables = 1 + net.bridge.bridge-nf-call-ip6tables = 1 + net.ipv4.ip_forward = 1 +- path: /var/lib/cloud/scripts/per-instance/kubeadm-join + permissions: '0755' + content: |+ + #!/bin/sh + + BASE_URL=https://dynk8s-provisioner.pyrocufflink.net + + instance_id=$(curl -s 169.254.169.254/latest/meta-data/instance-id) + az=$(curl -s 169.254.169.254/latest/meta-data/placement/availability-zone) + + curl -fs "${BASE_URL}"/wireguard/config/${instance_id} \ + -o /etc/wireguard/wg0.conf || exit + systemctl enable --now wg-quick@wg0 || exit + + modprobe br_netfilter || exit + sysctl -w -f /etc/sysctl.d/k8s.conf || exit + + swapoff -a || exit + touch /etc/systemd/zram-generator.conf || exit + systemctl daemon-reload || exit + systemctl stop 'systemd-zram-setup@*' || exit + + systemctl enable crio iscsid kubelet || exit + systemctl start crio iscsid || exit + + internal_ip=$( + ip address show dev wg0 primary | \ + sed -rn 's/.*inet ([0-9.]+).*/\1/p' + ) + + cat > /run/joinconfiguration <