Unverified Commit 20afd4f1 authored by Derek Nola's avatar Derek Nola
Browse files

Simplify K3s service startup for HA 



Signed-off-by: default avatarDerek Nola <derek.nola@suse.com>
parent 565c9fa0

Vagrantfile

+2 −2
Original line number Diff line number Diff line
@@ -28,10 +28,10 @@ def provision(vm, role, node_num) 
    ansible.extra_vars = {

      k3s_version: "v1.26.5+k3s1",

      api_endpoint: "#{NETWORK_PREFIX}.100",

      token: "myyagrant",

      token: "myvagrant",

      # Required to use the private network configured above

      extra_server_args: "--node-external-ip #{node_ip} --flannel-iface eth1", 

      extra_agent_args: "",

      extra_agent_args: "--node-external-ip #{node_ip} --flannel-iface eth1",

    }

  end

end

roles/k3s/agent/tasks/main.yml

+1 −1
Original line number Diff line number Diff line
@@ -12,5 +12,5 @@ 
  ansible.builtin.systemd:

    name: k3s-agent

    daemon_reload: true

    state: restarted

    state: started

    enabled: true

roles/k3s/agent/templates/k3s-agent.service.j2

+11 −6
Original line number Diff line number Diff line 
[Unit]

Description=Lightweight Kubernetes

Documentation=https://k3s.io

Wants=network-online.target

After=network-online.target



[Install]

WantedBy=multi-user.target



[Service]

Type=notify

ExecStartPre=-/sbin/modprobe br_netfilter

ExecStartPre=-/sbin/modprobe overlay

ExecStart=/usr/local/bin/k3s agent --data-dir {{ k3s_server_location }} --server https://{{ api_endpoint }}:{{ api_port }} --token {{ token }} {{ extra_agent_args }}

EnvironmentFile=-/etc/default/%N

EnvironmentFile=-/etc/sysconfig/%N

EnvironmentFile=-/etc/systemd/system/k3s.service.env

KillMode=process

Delegate=yes

# Having non-zero Limit*s causes performance problems due to accounting overhead
@@ -19,6 +23,7 @@ TasksMax=infinity 
TimeoutStartSec=0

Restart=always

RestartSec=5s



[Install]

WantedBy=multi-user.target
 
ExecStartPre=/bin/sh -xc '! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service'

ExecStartPre=-/sbin/modprobe br_netfilter

ExecStartPre=-/sbin/modprobe overlay

ExecStart=/usr/local/bin/k3s agent --data-dir {{ k3s_server_location }} --server https://{{ api_endpoint }}:{{ api_port }} --token {{ token }} {{ extra_agent_args }}
 
 No newline at end of file

roles/k3s/server/tasks/main.yml

+44 −44
Original line number Diff line number Diff line
@@ -2,22 +2,30 @@ 
- name: Init first server node

  when: ansible_hostname == groups['server'][0]

  block:

    - name: Start temporary service for HA cluster

      ansible.builtin.command:

        cmd: >

          systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server

          --cluster-init --token {{ token }} --tls-san {{ api_endpoint }} --data-dir {{ k3s_server_location }} {{ extra_server_args}}

        # noqa: jinja[spacing]

        creates: "{{ k3s_server_location }}/server/node-token"

    - name: Copy K3s service file [Single]

      when: groups['server'] | length == 1

      ansible.builtin.template:

        src: "k3s-single.service.j2"

        dest: "{{ systemd_dir }}/k3s.service"

        owner: root

        group: root

        mode: 0644



    - name: Copy K3s service file [HA]

      when: groups['server'] | length > 1

      ansible.builtin.template:

        src: "k3s-cluster-init.service.j2"

        dest: "{{ systemd_dir }}/k3s.service"

        owner: root

        group: root

        mode: 0644



    - name: Start temporary service for single server cluster

      ansible.builtin.command:

        cmd: >

          systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server

          --token {{ token }} --tls-san {{ api_endpoint }} --data-dir {{ k3s_server_location }} {{ extra_server_args }}

        creates: "{{ k3s_server_location }}/server/node-token"

      when: groups['server'] | length == 1

    - name: Enable and check K3s service

      ansible.builtin.systemd:

        name: k3s

        daemon_reload: true

        state: started

        enabled: true



    - name: Create directory .kube

      ansible.builtin.file:
@@ -26,6 +34,10 @@ 
        owner: "{{ ansible_user }}"

        mode: "u=rwx,g=rx,o="



    - name: Pause to allow server startup

      ansible.builtin.pause:

        seconds: 10



    - name: Copy config file to user home directory

      ansible.builtin.copy:

        src: /etc/rancher/k3s/k3s.yaml
@@ -48,15 +60,25 @@ 
        flat: true



- name: Start other server if any and verify status

  when:

    - (groups['server'] | length) > 1

    - ansible_hostname != groups['server'][0]

  block:

    - name: Init additonal server nodes

      ansible.builtin.command:

        cmd: >

          systemd-run -p RestartSec=2 -p Restart=on-failure --unit=k3s-init k3s server

          --token "{{ hostvars[groups['server'][0]]['token'] }}" --server https://{{ api_endpoint }}:{{ api_port }}

          --tls-san {{ api_endpoint }} --data-dir {{ k3s_server_location }} {{ extra_server_args }}

        creates: "{{ k3s_server_location }}/server/node-token"

      when: ansible_hostname != groups['server'][0]

    - name: Copy K3s service file [HA]

      when: groups['server'] | length > 1

      ansible.builtin.template:

        src: "k3s-ha.service.j2"

        dest: "{{ systemd_dir }}/k3s.service"

        owner: root

        group: root

        mode: 0644



    - name: Enable and check K3s service

      ansible.builtin.systemd:

        name: k3s

        daemon_reload: true

        state: started

        enabled: true



    - name: Verify that all server nodes joined

      when: (groups['server'] | length) > 1
@@ -68,28 +90,6 @@ 
      retries: 20

      delay: 10

      changed_when: false

  always:

    - name: Kill the temporary init service

      ansible.builtin.systemd:

        name: k3s-init

        state: stopped

      failed_when: false



- name: Copy K3s service file

  ansible.builtin.template:

    src: "k3s-server.service.j2"

    dest: "{{ systemd_dir }}/k3s-server.service"

    owner: root

    group: root

    mode: 0644

  register: k3s_service



- name: Enable and check K3s service

  ansible.builtin.systemd:

    name: k3s-server

    daemon_reload: true

    state: restarted

    enabled: true



- name: Create symlinks

  ansible.builtin.file:

roles/k3s/server/templates/k3s-cluster-init.service.j2

0 → 100644
+28 −0
Original line number Diff line number Diff line 
[Unit]

Description=Lightweight Kubernetes

Documentation=https://k3s.io

Wants=network-online.target

After=network-online.target



[Install]

WantedBy=multi-user.target



[Service]

Type=notify

EnvironmentFile=-/etc/default/%N

EnvironmentFile=-/etc/sysconfig/%N

EnvironmentFile=-/etc/systemd/system/k3s.service.env

KillMode=process

Delegate=yes

# Having non-zero Limit*s causes performance problems due to accounting overhead

# in the kernel. We recommend using cgroups to do container-local accounting.

LimitNOFILE=1048576

LimitNPROC=infinity

LimitCORE=infinity

TasksMax=infinity

TimeoutStartSec=0

Restart=always

RestartSec=5s

ExecStartPre=-/sbin/modprobe br_netfilter

ExecStartPre=-/sbin/modprobe overlay

ExecStart=/usr/local/bin/k3s server --cluster-init --data-dir {{ k3s_server_location }} --token {{ token }} {{ extra_server_args }} 
 No newline at end of file