diff --git a/.gitmodules b/.gitmodules index 3861c82..5d60873 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,18 +1,18 @@ [submodule "roles/ericomeehan.eom"] path = roles/ericomeehan.eom - url = git@git.eom.dev:eom/ansible-role-eom.git + url = git@git.eom.dev:DevOps/ansible-role-eom.git [submodule "roles/ericomeehan.debian"] path = roles/ericomeehan.debian - url = git@git.eom.dev:eom/ansible-role-debian.git + url = git@git.eom.dev:DevOps/ansible-role-debian.git [submodule "roles/ericomeehan.gondwana"] path = roles/ericomeehan.gondwana url = git@git.eom.dev:eom/ansible-role-gondwana.git [submodule "roles/ericomeehan.ericomeehan"] path = roles/ericomeehan.ericomeehan - url = git@git.eom.dev:eom/ansible-role-ericomeehan.git + url = git@git.eom.dev:eric/ansible-role-ericomeehan.git [submodule "roles/ericomeehan.nvidia_driver"] path = roles/ericomeehan.nvidia_driver - url = git@git.eom.dev:eom/ansible-role-nvidia-driver.git + url = git@git.eom.dev:DevOps/ansible-role-nvidia-driver.git [submodule "roles/ericomeehan.kraken-bot"] path = roles/ericomeehan.kraken-bot url = git@git.eom.dev:kraken/ansible-role-kraken-bot.git diff --git a/group_vars/control_planes.yaml b/group_vars/control_planes.yaml index 23acfd0..43a2994 100644 --- a/group_vars/control_planes.yaml +++ b/group_vars/control_planes.yaml @@ -1,6 +1,7 @@ --- # Group vars for Kubernetes control plane nodes kubernetes_role: control_plane +helm_version: 'v3.16.3' open_ports: - interface: any protocol: tcp diff --git a/host_vars/alpha-worker-10.yaml b/host_vars/alpha-worker-10.yaml new file mode 100644 index 0000000..4df892e --- /dev/null +++ b/host_vars/alpha-worker-10.yaml @@ -0,0 +1 @@ +name: alpha-worker-10 diff --git a/host_vars/alpha-worker-11.yaml b/host_vars/alpha-worker-11.yaml new file mode 100644 index 0000000..5f8a46a --- /dev/null +++ b/host_vars/alpha-worker-11.yaml @@ -0,0 +1 @@ +name: alpha-worker-11 diff --git a/host_vars/alpha-worker-12.yaml b/host_vars/alpha-worker-12.yaml new file mode 100644 index 0000000..ab70326 --- /dev/null +++ b/host_vars/alpha-worker-12.yaml @@ -0,0 +1 @@ +name: alpha-worker-12 diff --git a/host_vars/alpha-worker-4.yaml b/host_vars/alpha-worker-4.yaml new file mode 100644 index 0000000..d89ad7d --- /dev/null +++ b/host_vars/alpha-worker-4.yaml @@ -0,0 +1 @@ +name: alpha-worker-4 diff --git a/host_vars/alpha-worker-5.yaml b/host_vars/alpha-worker-5.yaml new file mode 100644 index 0000000..8146bce --- /dev/null +++ b/host_vars/alpha-worker-5.yaml @@ -0,0 +1 @@ +name: alpha-worker-5 diff --git a/host_vars/alpha-worker-6.yaml b/host_vars/alpha-worker-6.yaml new file mode 100644 index 0000000..0d17459 --- /dev/null +++ b/host_vars/alpha-worker-6.yaml @@ -0,0 +1 @@ +name: alpha-worker-6 diff --git a/host_vars/alpha-worker-7.yaml b/host_vars/alpha-worker-7.yaml new file mode 100644 index 0000000..443811d --- /dev/null +++ b/host_vars/alpha-worker-7.yaml @@ -0,0 +1 @@ +name: alpha-worker-7 diff --git a/host_vars/alpha-worker-8.yaml b/host_vars/alpha-worker-8.yaml new file mode 100644 index 0000000..f01bdb2 --- /dev/null +++ b/host_vars/alpha-worker-8.yaml @@ -0,0 +1 @@ +name: alpha-worker-8 diff --git a/host_vars/alpha-worker-9.yaml b/host_vars/alpha-worker-9.yaml new file mode 100644 index 0000000..d04d476 --- /dev/null +++ b/host_vars/alpha-worker-9.yaml @@ -0,0 +1 @@ +name: alpha-worker-9 diff --git a/host_vars/gamma-worker-0.yaml b/host_vars/gamma-worker-0.yaml deleted file mode 100644 index e6fb694..0000000 --- a/host_vars/gamma-worker-0.yaml +++ /dev/null @@ -1 +0,0 @@ -name: gamma-worker-0 diff --git a/host_vars/gamma-worker-1.yaml b/host_vars/gamma-worker-1.yaml deleted file mode 100644 index fa5c662..0000000 --- a/host_vars/gamma-worker-1.yaml +++ /dev/null @@ -1 +0,0 @@ -name: gamma-worker-1 diff --git a/host_vars/gamma-worker-2.yaml b/host_vars/gamma-worker-2.yaml deleted file mode 100644 index fba0cf1..0000000 --- a/host_vars/gamma-worker-2.yaml +++ /dev/null @@ -1 +0,0 @@ -name: gamma-worker-2 diff --git a/host_vars/gamma-worker-3.yaml b/host_vars/gamma-worker-3.yaml deleted file mode 100644 index 15a3ab8..0000000 --- a/host_vars/gamma-worker-3.yaml +++ /dev/null @@ -1 +0,0 @@ -name: gamma-worker-3 diff --git a/host_vars/poweredge-t640.yaml b/host_vars/poweredge-t640.yaml index 997c95a..ff06ab7 100644 --- a/host_vars/poweredge-t640.yaml +++ b/host_vars/poweredge-t640.yaml @@ -19,15 +19,14 @@ libvirt_guests: - alpha-worker-1 - alpha-worker-2 - alpha-worker-3 - - beta-control-plane - - beta-worker-0 - - beta-worker-1 - - beta-worker-2 - - beta-worker-3 - - gamma-control-plane - - gamma-worker-0 - - gamma-worker-1 - - gamma-worker-2 - - gamma-worker-3 + - alpha-worker-4 + - alpha-worker-5 + - alpha-worker-6 + - alpha-worker-7 + - alpha-worker-8 + - alpha-worker-9 + - alpha-worker-10 + - alpha-worker-11 + - alpha-worker-12 nfs_exports: ["/data *(rw,sync,no_root_squash)"] nvidia_driver_needed: true diff --git a/inventories/eom.dev.yml b/inventories/eom.dev.yml index 98e742a..f4c9513 100644 --- a/inventories/eom.dev.yml +++ b/inventories/eom.dev.yml @@ -19,22 +19,21 @@ all: control_planes: hosts: alpha-control-plane: - beta-control-plane: - gamma-control-plane: workers: hosts: alpha-worker-0: alpha-worker-1: alpha-worker-2: alpha-worker-3: - beta-worker-0: - beta-worker-1: - beta-worker-2: - beta-worker-3: - gamma-worker-0: - gamma-worker-1: - gamma-worker-2: - gamma-worker-3: + alpha-worker-4: + alpha-worker-5: + alpha-worker-6: + alpha-worker-7: + alpha-worker-8: + alpha-worker-9: + alpha-worker-10: + alpha-worker-11: + alpha-worker-12: alpha: hosts: alpha-control-plane: @@ -42,17 +41,12 @@ all: alpha-worker-1: alpha-worker-2: alpha-worker-3: - beta: - hosts: - beta-control-plane: - beta-worker-0: - beta-worker-1: - beta-worker-2: - beta-worker-3: - gamma: - hosts: - gamma-control-plane: - gamma-worker-0: - gamma-worker-1: - gamma-worker-2: - gamma-worker-3: + alpha-worker-4: + alpha-worker-5: + alpha-worker-6: + alpha-worker-7: + alpha-worker-8: + alpha-worker-9: + alpha-worker-10: + alpha-worker-11: + alpha-worker-12: diff --git a/main.yaml b/main.yaml index 2c2d0c4..435ea73 100644 --- a/main.yaml +++ b/main.yaml @@ -53,6 +53,7 @@ - qemu-utils - qemu-system-x86 - libvirt-daemon-system + - prometheus-libvirt-exporter - python3-libvirt - python3-lxml state: present diff --git a/reassign.yaml b/reassign.yaml new file mode 100644 index 0000000..0e5e39d --- /dev/null +++ b/reassign.yaml @@ -0,0 +1,154 @@ +- name: Deploy virtual machines + hosts: poweredge-t640 + become: true + vars_files: + - ../secrets.yaml + vars: + libvirt_guests: + - alpha-worker-8 + - alpha-worker-9 + - alpha-worker-10 + - alpha-worker-11 + - alpha-worker-12 + tasks: + - name: Download base image + get_url: + url: https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2 + dest: /var/lib/libvirt/images/debian-12-generic-amd64.qcow2 + force: true + + - name: Copy base image + copy: + src: /var/lib/libvirt/images/debian-12-generic-amd64.qcow2 + remote_src: true + dest: "/var/lib/libvirt/images/{{ item }}.qcow2" + force: true + loop: "{{ libvirt_guests }}" + + - name: Resize images + command: "qemu-img resize -f qcow2 /var/lib/libvirt/images/{{ item }}.qcow2 16G" + loop: "{{ libvirt_guests }}" + + - name: Create cloud-config directory + file: + path: "/tmp/{{ item }}" + state: directory + loop: "{{ libvirt_guests }}" + + - name: Copy cloud-config user-data template + template: + src: user-data.j2 + dest: "/tmp/{{ domain.name }}/user-data" + force: true + loop: "{{ libvirt_guests }}" + vars: + domain: "{{ hostvars[item] }}" + + - name: Copy cloud-config meta-data template + template: + src: meta-data.j2 + dest: "/tmp/{{ domain.name }}/meta-data" + force: true + loop: "{{ libvirt_guests }}" + vars: + domain: "{{ hostvars[item] }}" + + - name: Generate iso + command: "genisoimage -output /var/lib/libvirt/images/{{ item }}.iso -volid cidata -joliet -rock /tmp/{{ item }}/user-data /tmp/{{ item }}/meta-data" + loop: "{{ libvirt_guests }}" + + - name: Define libvirt virtual machine + community.libvirt.virt: + command: define + xml: "{{ lookup('template', 'libvirt-vm.xml.j2') }}" + loop: "{{ libvirt_guests }}" + vars: + domain: "{{ hostvars[item] }}" + + - name: Create libvirt virtual machine + community.libvirt.virt: + name: "{{ item }}" + command: create + loop: "{{ libvirt_guests }}" + + - name: Autostart libvirt virtual machines + community.libvirt.virt: + name: "{{ item }}" + autostart: true + loop: "{{ libvirt_guests }}" + + - name: Wait for guest initialization + wait_for: + timeout: 300 + + - name: Reset libvirt virtual machines for filesystem resize + command: "virsh reset {{ item }}" + loop: "{{ libvirt_guests }}" + +- name: Wait for manual tasks + hosts: localhost + tasks: + - name: Trust SSH identities + pause: + prompt: "Press Enter to continue..." + +- name: Initialize virtual machines + hosts: + - alpha-worker-8 + - alpha-worker-9 + - alpha-worker-10 + - alpha-worker-11 + - alpha-worker-12 + become: true + vars_files: + - ../secrets.yaml + roles: + - role: ericomeehan.ericomeehan + +- name: Initialize Kubernetes clusters + hosts: + - alpha-worker-8 + - alpha-worker-9 + - alpha-worker-10 + - alpha-worker-11 + - alpha-worker-12 + become: true + pre_tasks: + - name: Enable IPv4 packet forwarding + lineinfile: + path: /etc/sysctl.conf + line: 'net.ipv4.ip_forward = 1' + state: present + + - name: Enable IPv6 packet forwarding + lineinfile: + path: /etc/sysctl.conf + line: 'net.ipv6.conf.all.forwarding = 1' + state: present + + - name: Reload sysctl configuration + command: sysctl --system + + - name: Enable br_netfilter kernel module + command: modprobe br_netfilter + + - name: Add the module to a configuration file for persistence + lineinfile: + path: /etc/modules-load.d/modules.conf + line: "br_netfilter" + + - name: Install kubernetes library + apt: + name: python3-kubernetes + state: present + + roles: + - role: geerlingguy.containerd + +- name: Alpha Cluster + hosts: + - alpha + become: true + roles: + - role: geerlingguy.kubernetes + diff --git a/roles/ericomeehan.eom b/roles/ericomeehan.eom index 67778bb..e8178f2 160000 --- a/roles/ericomeehan.eom +++ b/roles/ericomeehan.eom @@ -1 +1 @@ -Subproject commit 67778bbe57120fb67dec9d019559cf911af3b0a6 +Subproject commit e8178f218d8fcbf58a46b928ee7451901e104125 diff --git a/roles/ericomeehan.nvidia_driver b/roles/ericomeehan.nvidia_driver index 4b9534a..7a3c04f 160000 --- a/roles/ericomeehan.nvidia_driver +++ b/roles/ericomeehan.nvidia_driver @@ -1 +1 @@ -Subproject commit 4b9534a27ef2c7a096f58959d9762647f524b96d +Subproject commit 7a3c04febc0ab5d6370da362ab80d862bfc3dd39