From 7026432c0e7cd275af032c554dcd99dad4e62b2e Mon Sep 17 00:00:00 2001 From: Pawel Niewiadomski <11110000b@gmail.com> Date: Sat, 13 Apr 2024 23:11:55 +0200 Subject: [PATCH 01/20] Use generic/ubuntu2204 box image --- settings.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.yaml b/settings.yaml index c4b50d2..48ca060 100644 --- a/settings.yaml +++ b/settings.yaml @@ -30,7 +30,7 @@ nodes: # - host_path: ../images # vm_path: /vagrant/images software: - box: bento/ubuntu-22.04 + box: generic/ubuntu2204 calico: 3.26.0 # To skip the dashboard installation, set its version to an empty value or comment it out: dashboard: 2.7.0 From 0bf44c62497c37d477434b970412e57eb37bd169 Mon Sep 17 00:00:00 2001 From: Pawel Niewiadomski <11110000b@gmail.com> Date: Sat, 13 Apr 2024 23:13:33 +0200 Subject: [PATCH 02/20] Updated README.md with an example of running this configuration with libvirt provider --- README.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f90b3bf..b238132 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,8 @@ Use code **SCRIPT20** to save $326 with the following bundle. ## Setup Prerequisites -- A working Vagrant setup using Vagrant + VirtualBox +- A working Vagrant setup using Vagrant with VirtualBox or libvirt available +- If using libvirt user needs to be added to libvirt group (`sudo usermod -aG libvirt $(whoami)`) ## Documentation @@ -47,6 +48,7 @@ Refer to this link for documentation full: https://devopscube.com/kubernetes-clu 2. 8 Gig + RAM workstation as the Vms use 3 vCPUS and 4+ GB RAM ## For MAC/Linux Users +## For MAC/Linux Users with VirtualBox The latest version of Virtualbox for Mac/Linux can cause issues. @@ -70,8 +72,15 @@ To provision the cluster, execute the following commands. ```shell git clone https://github.com/scriptcamp/vagrant-kubeadm-kubernetes.git cd vagrant-kubeadm-kubernetes -vagrant up +vagrant up --provider virtualbox +``` + +To use libvirt/KVM instead use: + +```shell +vagrant up --provider libvirt ``` + ## Set Kubeconfig file variable ```shell From b352f8486ebb240535a47524c8bc671822e2d876 Mon Sep 17 00:00:00 2001 From: Pawel Niewiadomski <11110000b@gmail.com> Date: Sat, 13 Apr 2024 23:14:13 +0200 Subject: [PATCH 03/20] Updated README.md with an example of setting firewall rules that will allow NFS access --- README.md | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b238132..a8ab098 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,34 @@ Refer to this link for documentation full: https://devopscube.com/kubernetes-clu 1. Working Vagrant setup 2. 8 Gig + RAM workstation as the Vms use 3 vCPUS and 4+ GB RAM -## For MAC/Linux Users + +## Firewall Configuration + +To allow virtual machines to access NFS server on the host machine you will probably need to update firewall rules. + +### Rules for VirtualBox + +```shell +sudo firewall-cmd --permanent --new-zone=virtualbox \ + && sudo firewall-cmd --permanent --zone=virtualbox --add-interface=vboxnet0 \ + && sudo firewall-cmd --permanent --zone=virtualbox --add-interface=vboxnet1 \ + && sudo firewall-cmd --permanent --zone=virtualbox --add-service=nfs3 \ + && sudo firewall-cmd --permanent --zone=virtualbox --add-service=nfs \ + && sudo firewall-cmd --permanent --zone=virtualbox --add-service=rpc-bind \ + && sudo firewall-cmd --permanent --zone=virtualbox --add-service=mountd \ + && sudo firewall-cmd --reload +``` + +### Rules for libvirt + +```shell +sudo firewall-cmd --permanent --zone=libvirt --add-service=nfs3 \ + && sudo firewall-cmd --permanent --zone=libvirt --add-service=nfs \ + && sudo firewall-cmd --permanent --zone=libvirt --add-service=rpc-bind \ + && sudo firewall-cmd --permanent --zone=libvirt --add-service=mountd \ + && sudo firewall-cmd --reload +``` + ## For MAC/Linux Users with VirtualBox The latest version of Virtualbox for Mac/Linux can cause issues. From 5ceba1f9e621abcfab06e519a90f39e526003909 Mon Sep 17 00:00:00 2001 From: Pawel Niewiadomski <11110000b@gmail.com> Date: Sat, 13 Apr 2024 23:19:57 +0200 Subject: [PATCH 04/20] Updated master.sh and node.sh to avoid race condition in libvirt when node can boot up faster than controlplane --- scripts/master.sh | 3 +-- scripts/node.sh | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/master.sh b/scripts/master.sh index 4582c92..6b9f7ae 100644 --- a/scripts/master.sh +++ b/scripts/master.sh @@ -29,10 +29,9 @@ else fi cp -i /etc/kubernetes/admin.conf $config_path/config -touch $config_path/join.sh -chmod +x $config_path/join.sh kubeadm token create --print-join-command > $config_path/join.sh +chmod +x $config_path/join.sh # Install Calico Network Plugin diff --git a/scripts/node.sh b/scripts/node.sh index a575417..34a00cc 100755 --- a/scripts/node.sh +++ b/scripts/node.sh @@ -6,6 +6,11 @@ set -euxo pipefail config_path="/vagrant/configs" +while [ ! -x $config_path/join.sh ]; do + echo "Waiting for master.sh to finish..." + sleep 15 +done + /bin/bash $config_path/join.sh -v sudo -i -u vagrant bash << EOF From a7463a56f4d471167b64a0f482d7c9ef5154415f Mon Sep 17 00:00:00 2001 From: Pawel Niewiadomski <11110000b@gmail.com> Date: Sat, 13 Apr 2024 23:21:44 +0200 Subject: [PATCH 05/20] Use NFS v4 for /vagrant mount --- Vagrantfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Vagrantfile b/Vagrantfile index 2803482..fa9f10a 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -11,6 +11,8 @@ IP_START = Integer(IP_SECTIONS.captures[1]) NUM_WORKER_NODES = settings["nodes"]["workers"]["count"] Vagrant.configure("2") do |config| + config.vm.synced_folder "./", "/vagrant", type: "nfs", nfs_version: 4 + config.vm.provision "shell", env: { "IP_NW" => IP_NW, "IP_START" => IP_START, "NUM_WORKER_NODES" => NUM_WORKER_NODES }, inline: <<-SHELL apt-get update -y echo "$IP_NW$((IP_START)) controlplane" >> /etc/hosts From 0d5b421f06917e677d62ada064640dc48e611bd1 Mon Sep 17 00:00:00 2001 From: Pawel Niewiadomski <11110000b@gmail.com> Date: Sat, 13 Apr 2024 23:25:49 +0200 Subject: [PATCH 06/20] Separated vagrant invocation to be more explicit --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 97d8a19..37fecf8 100644 --- a/README.md +++ b/README.md @@ -98,10 +98,15 @@ To provision the cluster, execute the following commands. ```shell git clone https://github.com/scriptcamp/vagrant-kubeadm-kubernetes.git cd vagrant-kubeadm-kubernetes +``` + +### With VirtualBox + +```shell vagrant up --provider virtualbox ``` -To use libvirt/KVM instead use: +### With libvirt ```shell vagrant up --provider libvirt From 7e6c250681aedbd99c33d5ad446d34e035f27168 Mon Sep 17 00:00:00 2001 From: Pawel Niewiadomski <11110000b@gmail.com> Date: Sun, 14 Apr 2024 07:29:22 +0200 Subject: [PATCH 07/20] Updated Vagrantfile with libvirt configuration --- Vagrantfile | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Vagrantfile b/Vagrantfile index fa9f10a..06ac9c7 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -13,6 +13,11 @@ NUM_WORKER_NODES = settings["nodes"]["workers"]["count"] Vagrant.configure("2") do |config| config.vm.synced_folder "./", "/vagrant", type: "nfs", nfs_version: 4 + config.vm.provider "libvirt" do |libvirt| + libvirt.uri = "qemu:///system" + libvirt.driver = "qemu" + end + config.vm.provision "shell", env: { "IP_NW" => IP_NW, "IP_START" => IP_START, "NUM_WORKER_NODES" => NUM_WORKER_NODES }, inline: <<-SHELL apt-get update -y echo "$IP_NW$((IP_START)) controlplane" >> /etc/hosts @@ -36,6 +41,7 @@ Vagrant.configure("2") do |config| controlplane.vm.synced_folder shared_folder["host_path"], shared_folder["vm_path"] end end + controlplane.vm.provider "virtualbox" do |vb| vb.cpus = settings["nodes"]["control"]["cpu"] vb.memory = settings["nodes"]["control"]["memory"] @@ -43,6 +49,12 @@ Vagrant.configure("2") do |config| vb.customize ["modifyvm", :id, "--groups", ("/" + settings["cluster_name"])] end end + + controlplane.vm.provider "libvirt" do |vb| + vb.cpus = settings["nodes"]["control"]["cpu"] + vb.memory = settings["nodes"]["control"]["memory"] + end + controlplane.vm.provision "shell", env: { "DNS_SERVERS" => settings["network"]["dns_servers"].join(" "), @@ -72,6 +84,7 @@ Vagrant.configure("2") do |config| node.vm.synced_folder shared_folder["host_path"], shared_folder["vm_path"] end end + node.vm.provider "virtualbox" do |vb| vb.cpus = settings["nodes"]["workers"]["cpu"] vb.memory = settings["nodes"]["workers"]["memory"] @@ -79,6 +92,12 @@ Vagrant.configure("2") do |config| vb.customize ["modifyvm", :id, "--groups", ("/" + settings["cluster_name"])] end end + + node.vm.provider "libvirt" do |vb| + vb.cpus = settings["nodes"]["workers"]["cpu"] + vb.memory = settings["nodes"]["workers"]["memory"] + end + node.vm.provision "shell", env: { "DNS_SERVERS" => settings["network"]["dns_servers"].join(" "), From 8a9ef923007e394f6f25e5829a89a54ce1fa853a Mon Sep 17 00:00:00 2001 From: Pawel Niewiadomski <11110000b@gmail.com> Date: Sun, 14 Apr 2024 07:40:45 +0200 Subject: [PATCH 08/20] Simplified setting kubectl config --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 37fecf8..842f127 100644 --- a/README.md +++ b/README.md @@ -116,14 +116,14 @@ vagrant up --provider libvirt ```shell cd vagrant-kubeadm-kubernetes -cd configs -export KUBECONFIG=$(pwd)/config + +export KUBECONFIG=$(pwd)/configs/config ``` or you can copy the config file to .kube directory. ```shell -cp config ~/.kube/ +cp configs/config ~/.kube/ ``` ## Install Kubernetes Dashboard From c1869c65dfc22af4a136f6e7b9ef14b50381cfae Mon Sep 17 00:00:00 2001 From: Pawel Niewiadomski <11110000b@gmail.com> Date: Thu, 11 Jul 2024 14:04:14 +0200 Subject: [PATCH 09/20] It's possible to mark the domain as automatically started on the boot time --- Vagrantfile | 11 +++++++++-- settings.yaml | 8 +++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 06ac9c7..d0f038a 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -2,6 +2,7 @@ require "yaml" vagrant_root = File.dirname(File.expand_path(__FILE__)) settings = YAML.load_file "#{vagrant_root}/settings.yaml" +autostart = settings["autostart"] IP_SECTIONS = settings["network"]["control_ip"].match(/^([0-9.]+\.)([^.]+)$/) # First 3 octets including the trailing dot: @@ -16,6 +17,8 @@ Vagrant.configure("2") do |config| config.vm.provider "libvirt" do |libvirt| libvirt.uri = "qemu:///system" libvirt.driver = "qemu" + libvirt.management_network_autostart = autostart + libvirt.autostart = autostart end config.vm.provision "shell", env: { "IP_NW" => IP_NW, "IP_START" => IP_START, "NUM_WORKER_NODES" => NUM_WORKER_NODES }, inline: <<-SHELL @@ -35,7 +38,8 @@ Vagrant.configure("2") do |config| config.vm.define "controlplane" do |controlplane| controlplane.vm.hostname = "controlplane" - controlplane.vm.network "private_network", ip: settings["network"]["control_ip"] + controlplane.vm.network "private_network", ip: settings["network"]["control_ip"], autostart: autostart + if settings["shared_folders"] settings["shared_folders"].each do |shared_folder| controlplane.vm.synced_folder shared_folder["host_path"], shared_folder["vm_path"] @@ -64,6 +68,7 @@ Vagrant.configure("2") do |config| "OS" => settings["software"]["os"] }, path: "scripts/common.sh" + controlplane.vm.provision "shell", env: { "CALICO_VERSION" => settings["software"]["calico"], @@ -78,7 +83,8 @@ Vagrant.configure("2") do |config| config.vm.define "node0#{i}" do |node| node.vm.hostname = "node0#{i}" - node.vm.network "private_network", ip: IP_NW + "#{IP_START + i}" + node.vm.network "private_network", ip: IP_NW + "#{IP_START + i}", autostart: autostart + if settings["shared_folders"] settings["shared_folders"].each do |shared_folder| node.vm.synced_folder shared_folder["host_path"], shared_folder["vm_path"] @@ -107,6 +113,7 @@ Vagrant.configure("2") do |config| "OS" => settings["software"]["os"] }, path: "scripts/common.sh" + node.vm.provision "shell", path: "scripts/node.sh" # Only install the dashboard after provisioning the last worker (and when enabled). diff --git a/settings.yaml b/settings.yaml index 48ca060..470a4df 100644 --- a/settings.yaml +++ b/settings.yaml @@ -1,6 +1,8 @@ --- # cluster_name is used to group the nodes in a folder within VirtualBox: cluster_name: Kubernetes Cluster +# Tell libvirt to automatically start machines when host is booting up +autostart: true # Uncomment to set environment variables for services such as crio and kubelet. # For example, configure the cluster to pull images via a proxy. # environment: | @@ -21,8 +23,8 @@ nodes: cpu: 2 memory: 4096 workers: - count: 2 - cpu: 1 + count: 3 + cpu: 2 memory: 2048 # Mount additional shared folders from the host into each virtual machine. # Note that the project directory is automatically mounted at /vagrant. @@ -33,7 +35,7 @@ software: box: generic/ubuntu2204 calico: 3.26.0 # To skip the dashboard installation, set its version to an empty value or comment it out: - dashboard: 2.7.0 + dashboard: kubernetes: 1.29.0-* os: xUbuntu_22.04 From 1de1254c7d036d66fb3c75dfa3b534c066caac8d Mon Sep 17 00:00:00 2001 From: Pawel Niewiadomski <11110000b@gmail.com> Date: Sun, 15 Sep 2024 14:35:27 +0200 Subject: [PATCH 10/20] Update kubernetes version --- settings.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/settings.yaml b/settings.yaml index 470a4df..37866f4 100644 --- a/settings.yaml +++ b/settings.yaml @@ -36,6 +36,5 @@ software: calico: 3.26.0 # To skip the dashboard installation, set its version to an empty value or comment it out: dashboard: - kubernetes: 1.29.0-* - + kubernetes: 1.31.1-* os: xUbuntu_22.04 From b8d87ccd7774ec4547521866fc233d92c10ced79 Mon Sep 17 00:00:00 2001 From: Pawel Niewiadomski <11110000b@gmail.com> Date: Sun, 15 Sep 2024 19:28:13 +0200 Subject: [PATCH 11/20] Fixed a typo --- scripts/common.sh | 2 +- settings.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/common.sh b/scripts/common.sh index c261cc7..206cdf8 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -20,7 +20,7 @@ sudo systemctl restart systemd-resolved # disable swap sudo swapoff -a -# keeps the swaf off during reboot +# keeps the swap off during reboot (crontab -l 2>/dev/null; echo "@reboot /sbin/swapoff -a") | crontab - || true sudo apt-get update -y diff --git a/settings.yaml b/settings.yaml index 37866f4..c56a91a 100644 --- a/settings.yaml +++ b/settings.yaml @@ -12,20 +12,20 @@ autostart: true # All IPs/CIDRs should be private and allowed in /etc/vbox/networks.conf. network: # Worker IPs are simply incremented from the control IP. - control_ip: 10.0.0.10 + control_ip: 10.1.0.10 dns_servers: - 8.8.8.8 - 1.1.1.1 - pod_cidr: 172.16.1.0/16 - service_cidr: 172.17.1.0/18 + pod_cidr: 172.16.2.0/16 + service_cidr: 172.17.2.0/18 nodes: control: - cpu: 2 + cpu: 6 memory: 4096 workers: count: 3 cpu: 2 - memory: 2048 + memory: 4096 # Mount additional shared folders from the host into each virtual machine. # Note that the project directory is automatically mounted at /vagrant. # shared_folders: From 81874f5d1dde25056f64d624003a441c25784b6c Mon Sep 17 00:00:00 2001 From: Pawel Niewiadomski <11110000b@gmail.com> Date: Wed, 18 Sep 2024 10:25:49 +0200 Subject: [PATCH 12/20] Moved settings.yaml to settings.yaml.sample so can easily customize it without worring it will get back to the repo --- .gitignore | 1 + README.md | 1 + settings.yaml => settings.yaml.sample | 0 3 files changed, 2 insertions(+) rename settings.yaml => settings.yaml.sample (100%) diff --git a/.gitignore b/.gitignore index 582c81c..bb27b19 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # General .vagrant/ configs/ +settings.yaml # Log files (if you are creating logs in debug mode, uncomment this) # *.log diff --git a/README.md b/README.md index 842f127..b8778f2 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,7 @@ To provision the cluster, execute the following commands. ```shell git clone https://github.com/scriptcamp/vagrant-kubeadm-kubernetes.git cd vagrant-kubeadm-kubernetes +cp settings.yaml.sample settings.yaml ``` ### With VirtualBox diff --git a/settings.yaml b/settings.yaml.sample similarity index 100% rename from settings.yaml rename to settings.yaml.sample From 04077da8905703b4b020bb39526a71d3d1ede58e Mon Sep 17 00:00:00 2001 From: Pawel Niewiadomski <11110000b@gmail.com> Date: Wed, 18 Sep 2024 10:40:17 +0200 Subject: [PATCH 13/20] Removed unnecessary sudo --- scripts/common.sh | 30 ++++++++++++++++-------------- scripts/master.sh | 8 ++++---- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/scripts/common.sh b/scripts/common.sh index 206cdf8..71c0226 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -8,45 +8,43 @@ set -euxo pipefail # DNS Setting if [ ! -d /etc/systemd/resolved.conf.d ]; then - sudo mkdir /etc/systemd/resolved.conf.d/ + mkdir /etc/systemd/resolved.conf.d/ fi -cat </dev/null; echo "@reboot /sbin/swapoff -a") | crontab - || true -sudo apt-get update -y +apt-get update -y # Create the .conf file to load the modules at bootup -cat < Date: Wed, 18 Sep 2024 10:42:08 +0200 Subject: [PATCH 14/20] Fixed failing jq --- scripts/common.sh | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/scripts/common.sh b/scripts/common.sh index 71c0226..f32b989 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -66,10 +66,30 @@ curl -fsSL https://pkgs.k8s.io/core:/stable:/v$KUBERNETES_VERSION_SHORT/deb/Rele echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v$KUBERNETES_VERSION_SHORT/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list -sudo apt-get update -y -sudo apt-get install -y kubelet="$KUBERNETES_VERSION" kubectl="$KUBERNETES_VERSION" kubeadm="$KUBERNETES_VERSION" -sudo apt-get update -y -sudo apt-get install -y jq + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | + tee /etc/apt/sources.list.d/docker.list + + # Update and install containerd + apt-get update -y + apt-get install -y containerd.io + + mkdir -p /etc/containerd + containerd config default > /etc/containerd/config.toml + + systemctl daemon-reload + systemctl enable containerd --now + systemctl start containerd + + apt-mark hold containerd.io + + echo "containerd runtime installed successfully" +fi + +mkdir -p /etc/apt/keyrings +curl -fsSL https://pkgs.k8s.io/core:/stable:/v$KUBERNETES_VERSION_SHORT/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg +echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v$KUBERNETES_VERSION_SHORT/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + + apt-get update -y apt-get install -y kubelet="$KUBERNETES_VERSION" kubectl="$KUBERNETES_VERSION" kubeadm="$KUBERNETES_VERSION" apt-get update -y @@ -78,8 +98,8 @@ apt-get install -y jq # Disable auto-update services apt-mark hold kubelet kubectl kubeadm +local_ip="$(ip --json a s | jq -r '.[] | select(.ifname == "eth1") | .addr_info[] | select(.family == "inet") | .local')" -local_ip="$(ip --json a s | jq -r '.[] | if .ifname == "eth1" then .addr_info[] | if .family == "inet" then .local else empty end else empty end')" cat > /etc/default/kubelet << EOF KUBELET_EXTRA_ARGS=--node-ip=$local_ip ${ENVIRONMENT} From 466b11f4ca461ed8b41b4db4c745a4b4a2587976 Mon Sep 17 00:00:00 2001 From: Pawel Niewiadomski <11110000b@gmail.com> Date: Wed, 18 Sep 2024 10:43:43 +0200 Subject: [PATCH 15/20] You can pick CRI implementation (containerd version doesn't work ATM for some reason) + you can also pick cri-o version --- Vagrantfile | 10 ++++++++-- scripts/common.sh | 34 +++++++++++++++++++++------------- settings.yaml.sample | 3 +++ 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index d0f038a..96ac1ba 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -65,7 +65,10 @@ Vagrant.configure("2") do |config| "ENVIRONMENT" => settings["environment"], "KUBERNETES_VERSION" => settings["software"]["kubernetes"], "KUBERNETES_VERSION_SHORT" => settings["software"]["kubernetes"][0..3], - "OS" => settings["software"]["os"] + "OS" => settings["software"]["os"], + "CRI" => settings["software"]["cri"], + "CRI_VERSION" => settings["software"]["cri-version"], + "CRI_VERSION_SHORT" => settings["software"]["cri-version"][0..3], }, path: "scripts/common.sh" @@ -110,7 +113,10 @@ Vagrant.configure("2") do |config| "ENVIRONMENT" => settings["environment"], "KUBERNETES_VERSION" => settings["software"]["kubernetes"], "KUBERNETES_VERSION_SHORT" => settings["software"]["kubernetes"][0..3], - "OS" => settings["software"]["os"] + "OS" => settings["software"]["os"], + "CRI" => settings["software"]["cri"], + "CRI_VERSION" => settings["software"]["cri-version"], + "CRI_VERSION_SHORT" => settings["software"]["cri-version"][0..3], }, path: "scripts/common.sh" diff --git a/scripts/common.sh b/scripts/common.sh index f32b989..062fc3f 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -47,24 +47,32 @@ sysctl --system apt-get update -y apt-get install -y software-properties-common curl apt-transport-https ca-certificates -curl -fsSL https://pkgs.k8s.io/addons:/cri-o:/prerelease:/main/deb/Release.key | - gpg --dearmor -o /etc/apt/keyrings/cri-o-apt-keyring.gpg -echo "deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/prerelease:/main/deb/ /" | - tee /etc/apt/sources.list.d/cri-o.list +if [ "$CRI" = "cri-o" ]; then + ## Install CRIO Runtime -sudo apt-get update -y -sudo apt-get install -y cri-o + curl -fsSL https://pkgs.k8s.io/addons:/cri-o:/stable:/v$CRI_VERSION_SHORT/deb/Release.key | + gpg --dearmor -o /etc/apt/keyrings/cri-o-apt-keyring.gpg -sudo systemctl daemon-reload -sudo systemctl enable crio --now -sudo systemctl start crio.service + echo "deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/stable:/v$CRI_VERSION_SHORT/deb/ /" | + tee /etc/apt/sources.list.d/cri-o.list -echo "CRI runtime installed successfully" + apt-get update -y + apt-get install -y cri-o="$CRI_VERSION" + + systemctl daemon-reload + systemctl enable crio --now + systemctl start crio.service + + apt-mark hold cri-o + + echo "CRI-O runtime installed successfully" +fi -sudo mkdir -p /etc/apt/keyrings -curl -fsSL https://pkgs.k8s.io/core:/stable:/v$KUBERNETES_VERSION_SHORT/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg -echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v$KUBERNETES_VERSION_SHORT/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list +if [ "$CRI" = "containerd" ]; then + ## Install containerd Runtime + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | + gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list diff --git a/settings.yaml.sample b/settings.yaml.sample index c56a91a..3bc0ca9 100644 --- a/settings.yaml.sample +++ b/settings.yaml.sample @@ -38,3 +38,6 @@ software: dashboard: kubernetes: 1.31.1-* os: xUbuntu_22.04 + # Pick either cri-o or containerd + cri: cri-o + cri-version: 1.30.4-* \ No newline at end of file From b5bd7a809788abce8ec5345ff9819ca587fd9799 Mon Sep 17 00:00:00 2001 From: Pawel Niewiadomski <11110000b@gmail.com> Date: Wed, 18 Sep 2024 11:51:28 +0200 Subject: [PATCH 16/20] Added an example how autostart for VirtualBox could be configured but it doesn't work - vboxautostart-service.service is crashing, probably only one VM per user is allowed --- Vagrantfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Vagrantfile b/Vagrantfile index 96ac1ba..9f0623c 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -49,9 +49,15 @@ Vagrant.configure("2") do |config| controlplane.vm.provider "virtualbox" do |vb| vb.cpus = settings["nodes"]["control"]["cpu"] vb.memory = settings["nodes"]["control"]["memory"] + if settings["cluster_name"] and settings["cluster_name"] != "" vb.customize ["modifyvm", :id, "--groups", ("/" + settings["cluster_name"])] end + + if autostart + vb.customize ["modifyvm", :id, "--autostart-enabled", "on"] + vb.customize ["modifyvm", :id, "--autostop-type", "acpishutdown"] + end end controlplane.vm.provider "libvirt" do |vb| @@ -97,9 +103,15 @@ Vagrant.configure("2") do |config| node.vm.provider "virtualbox" do |vb| vb.cpus = settings["nodes"]["workers"]["cpu"] vb.memory = settings["nodes"]["workers"]["memory"] + if settings["cluster_name"] and settings["cluster_name"] != "" vb.customize ["modifyvm", :id, "--groups", ("/" + settings["cluster_name"])] end + + if autostart + vb.customize ["modifyvm", :id, "--autostart-enabled", "on"] + vb.customize ["modifyvm", :id, "--autostop-type", "acpishutdown"] + end end node.vm.provider "libvirt" do |vb| From fde19980e48245879b65aed9fab6deca29afe0b0 Mon Sep 17 00:00:00 2001 From: Pawel Niewiadomski <11110000b@gmail.com> Date: Thu, 26 Sep 2024 20:31:36 +0200 Subject: [PATCH 17/20] Ignore any additional customization scripts --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index bb27b19..27eebf4 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,6 @@ Icon .AppleDesktop Network Trash Folder Temporary Items -.apdisk \ No newline at end of file +.apdisk + +scripts/customize-*.sh \ No newline at end of file From 9e0b750447f59412ac304af720e1abd4c8c586d2 Mon Sep 17 00:00:00 2001 From: Pawel Niewiadomski <11110000b@gmail.com> Date: Thu, 26 Sep 2024 20:32:05 +0200 Subject: [PATCH 18/20] Check if /vagrant has been correctly mounted through NFS --- scripts/common.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/common.sh b/scripts/common.sh index 062fc3f..d1e684a 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -4,6 +4,12 @@ set -euxo pipefail +# Check if /vagrant is correctly mounted through NFS +if ! mount | grep -q 'on /vagrant type nfs'; then + echo "/vagrant is not mounted through NFS, check your firewall settings" + exit 1 +fi + # Variable Declaration # DNS Setting From ecb6eb8af8af85ce28c20f5e4649a029961ebe32 Mon Sep 17 00:00:00 2001 From: Pawel Niewiadomski <11110000b@gmail.com> Date: Thu, 26 Sep 2024 20:32:37 +0200 Subject: [PATCH 19/20] Added support for additional node customization (using scripts/customize-node.sh) --- Vagrantfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Vagrantfile b/Vagrantfile index 9f0623c..2ecf835 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -134,6 +134,10 @@ Vagrant.configure("2") do |config| node.vm.provision "shell", path: "scripts/node.sh" + if File.exist?("scripts/customize-node.sh") + node.vm.provision "shell", path: "scripts/customize-node.sh" + end + # Only install the dashboard after provisioning the last worker (and when enabled). if i == NUM_WORKER_NODES and settings["software"]["dashboard"] and settings["software"]["dashboard"] != "" node.vm.provision "shell", path: "scripts/dashboard.sh" From 55447caaeeecbb4d1a608be0824001b707a078d9 Mon Sep 17 00:00:00 2001 From: Pawel Niewiadomski <11110000b@gmail.com> Date: Sat, 5 Oct 2024 16:05:00 +0200 Subject: [PATCH 20/20] Turn on config dir support --- scripts/common.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/common.sh b/scripts/common.sh index d1e684a..cd5b516 100644 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -114,7 +114,8 @@ apt-mark hold kubelet kubectl kubeadm local_ip="$(ip --json a s | jq -r '.[] | select(.ifname == "eth1") | .addr_info[] | select(.family == "inet") | .local')" +mkdir -p /etc/kubernetes/kubelet.conf.d cat > /etc/default/kubelet << EOF -KUBELET_EXTRA_ARGS=--node-ip=$local_ip +KUBELET_EXTRA_ARGS="--node-ip=$local_ip --config-dir=/etc/kubernetes/kubelet.conf.d" ${ENVIRONMENT} EOF