k8s-control-node.yml

## template: jinja
#cloud-config
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

---
users:
  - name: cloud
    sudo: ALL=(ALL) NOPASSWD:ALL
    shell: /bin/bash
    ssh_authorized_keys:
      {{ k8s.ssh.pub.key }}

write_files:
  - path: /etc/conf.d/nfs
    permissions: '0644'
    content: |
      OPTS_RPC_MOUNTD=""

  - path: /etc/kubernetes/pki/cloudstack/ca.crt
    permissions: '0644'
    content: |
      {{ k8s_control_node.ca.crt }}

  - path: /etc/kubernetes/pki/cloudstack/apiserver.crt
    permissions: '0644'
    content: |
      {{ k8s_control_node.apiserver.crt }}

  - path: /etc/kubernetes/pki/cloudstack/apiserver.key
    permissions: '0600'
    content: |
      {{ k8s_control_node.apiserver.key }}

  - path: /opt/bin/setup-kube-system
    permissions: '0700'
    owner: root:root
    content: |
      #!/bin/bash -e

      if [[ -f "/home/cloud/success" ]]; then
      echo "Already provisioned!"
      exit 0
      fi

      sysctl net.ipv4.conf.default.arp_announce=0
      sysctl net.ipv4.conf.default.arp_ignore=0
      sysctl net.ipv4.conf.all.arp_announce=0
      sysctl net.ipv4.conf.all.arp_ignore=0
      sysctl net.ipv4.conf.eth0.arp_announce=0 || sysctl net.ipv4.conf.ens35.arp_announce=0 || true
      sysctl net.ipv4.conf.eth0.arp_ignore=0 || sysctl net.ipv4.conf.ens35.arp_ignore=0 || true
      sed -i "s/net.ipv4.conf.default.arp_announce =.*$/net.ipv4.conf.default.arp_announce = 0/" /etc/sysctl.conf
      sed -i "s/net.ipv4.conf.default.arp_ignore =.*$/net.ipv4.conf.default.arp_ignore = 0/" /etc/sysctl.conf
      sed -i "s/net.ipv4.conf.all.arp_announce =.*$/net.ipv4.conf.all.arp_announce = 0/" /etc/sysctl.conf
      sed -i "s/net.ipv4.conf.all.arp_ignore =.*$/net.ipv4.conf.all.arp_ignore = 0/" /etc/sysctl.conf

      ISO_MOUNT_DIR=/mnt/k8sdisk
      BINARIES_DIR=${ISO_MOUNT_DIR}/
      K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/
      ATTEMPT_ONLINE_INSTALL=false
      setup_complete=false

      OFFLINE_INSTALL_ATTEMPT_SLEEP={{ k8s.install.wait.time }}
      MAX_OFFLINE_INSTALL_ATTEMPTS={{ k8s.install.reattempts.count }}
      if [[ -z $OFFLINE_INSTALL_ATTEMPT_SLEEP || $OFFLINE_INSTALL_ATTEMPT_SLEEP -eq 0 ]]; then
        OFFLINE_INSTALL_ATTEMPT_SLEEP=15
      fi
      if [[ -z $MAX_OFFLINE_INSTALL_ATTEMPTS || $MAX_OFFLINE_INSTALL_ATTEMPTS -eq 0 ]]; then
        MAX_OFFLINE_INSTALL_ATTEMPTS=100
      fi
      offline_attempts=1
      MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3
      EJECT_ISO_FROM_OS={{ k8s.eject.iso }}
      crucial_cmd_attempts=1
      iso_drive_path=""
      while true; do
        if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then
          echo "Warning: Offline install timed out!"
          break
        fi
        set +e
        output=`blkid -o device -t LABEL=CDROM`
        set -e
        if [ "$output" != "" ]; then
          while read -r line; do
            if [ ! -d "${ISO_MOUNT_DIR}" ]; then
              mkdir "${ISO_MOUNT_DIR}"
            fi
            retval=0
            set +e
            mount -o ro "${line}" "${ISO_MOUNT_DIR}"
            retval=$?
            set -e
            if [ $retval -eq 0 ]; then
              if [ -d "$BINARIES_DIR" ]; then
                iso_drive_path="${line}"
                break
              else
                umount "${line}" && rmdir "${ISO_MOUNT_DIR}"
              fi
            fi
          done <<< "$output"
        fi
        if [ -d "$BINARIES_DIR" ]; then
          break
        fi
        echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts"
        sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP
        offline_attempts=$[$offline_attempts + 1]
      done

      if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then
        export PATH=$PATH:/opt/bin
      fi

      if [ -d "$BINARIES_DIR" ]; then
        ### Binaries available offline ###
        echo "Installing binaries from ${BINARIES_DIR}"
        mkdir -p /opt/cni/bin
        tar -f "${BINARIES_DIR}/cni/cni-plugins-"*64.tgz -C /opt/cni/bin -xz

        mkdir -p /opt/bin
        tar -f "${BINARIES_DIR}/cri-tools/crictl-linux-"*64.tar.gz -C /opt/bin -xz

        mkdir -p /opt/bin
        cd /opt/bin
        cp -a ${BINARIES_DIR}/k8s/{kubeadm,kubelet,kubectl} .
        chmod +x {kubeadm,kubelet,kubectl}

        sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/kubelet.service > /etc/systemd/system/kubelet.service
        mkdir -p /etc/systemd/system/kubelet.service.d
        sed "s:/usr/bin:/opt/bin:g" ${BINARIES_DIR}/10-kubeadm.conf > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf

        echo "KUBELET_EXTRA_ARGS=--cgroup-driver=systemd" > /etc/default/kubelet

        output=`ls ${BINARIES_DIR}/docker/`
        if [ "$output" != "" ]; then
          while read -r line; do
            crucial_cmd_attempts=1
            while true; do
              if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then
                echo "Loading docker image ${BINARIES_DIR}/docker/$line failed!"
                break;
              fi
              retval=0
              set +e
              ctr -n k8s.io image import "${BINARIES_DIR}/docker/$line"
              retval=$?
              set -e
              if [ $retval -eq 0 ]; then
                break;
              fi
              crucial_cmd_attempts=$[$crucial_cmd_attempts + 1]
            done
          done <<< "$output"
          setup_complete=true
        fi
        mkdir -p "${K8S_CONFIG_SCRIPTS_COPY_DIR}"
        cp ${BINARIES_DIR}/*.yaml "${K8S_CONFIG_SCRIPTS_COPY_DIR}"
        if [ -e "${BINARIES_DIR}/autoscaler.yaml" ]; then
          mkdir -p /opt/autoscaler
          cp "${BINARIES_DIR}/autoscaler.yaml" /opt/autoscaler/autoscaler_tmpl.yaml
        fi
        if [ -e "${BINARIES_DIR}/provider.yaml" ]; then
          mkdir -p /opt/provider
          cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml
        fi
        if [ -e "${BINARIES_DIR}/snapshot-crds.yaml" ]; then
          mkdir -p /opt/csi
          cp "${BINARIES_DIR}/snapshot-crds.yaml" /opt/csi/snapshot-crds.yaml
          cp "${BINARIES_DIR}/manifest.yaml" /opt/csi/manifest.yaml
        fi

        PAUSE_IMAGE=`ctr -n k8s.io images ls -q | grep "pause" | sort | tail -n 1`
        echo $PAUSE_IMAGE
        if [ -n "$PAUSE_IMAGE" ]; then
          sed -i "s|sandbox_image = .*|sandbox_image = \"$PAUSE_IMAGE\"|g" /etc/containerd/config.toml
        fi
        systemctl daemon-reload
        systemctl restart containerd

        umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}"
        if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then
          eject "${iso_drive_path}"
        fi
      fi
      if [ "$setup_complete" = false ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then
        ###  Binaries not available offline ###
        RELEASE="v1.16.3"
        CNI_VERSION="v0.7.5"
        CRICTL_VERSION="v1.16.0"
        echo "Warning: ${BINARIES_DIR} not found. Will get binaries and docker images from Internet."
        mkdir -p /opt/cni/bin
        curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-amd64-${CNI_VERSION}.tgz" | tar -C /opt/cni/bin -xz

        mkdir -p /opt/bin
        curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-amd64.tar.gz" | tar -C /opt/bin -xz

        mkdir -p /opt/bin
        cd /opt/bin
        curl -L --remote-name-all https://storage.googleapis.com/kubernetes-release/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl}
        chmod +x {kubeadm,kubelet,kubectl}

        curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service
        mkdir -p /etc/systemd/system/kubelet.service.d
        curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
      fi

      systemctl enable kubelet && systemctl start kubelet
      modprobe overlay && modprobe br_netfilter && sysctl net.bridge.bridge-nf-call-iptables=1

      if [ -d "$BINARIES_DIR" ] && [ "$ATTEMPT_ONLINE_INSTALL" = true ]; then
        crucial_cmd_attempts=1
        while true; do
          if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then
            echo "Warning: kubeadm pull images failed after multiple tries!"
            break;
          fi
          retval=0
          set +e
          kubeadm config images pull --cri-socket /run/containerd/containerd.sock
          retval=$?
          set -e
          if [ $retval -eq 0 ]; then
            break;
          fi
          crucial_cmd_attempts=$[$crucial_cmd_attempts + 1]
        done
      fi

  - path: /etc/kubernetes/kubeadm-config.yaml
    permissions: '0644'
    owner: root:root
    content: |
      apiVersion: kubeadm.k8s.io/v1beta3
      kind: ClusterConfiguration
      apiServer:
        certSANs:
        {{ k8s_control.server_ips }}
      controlPlaneEndpoint: {{ k8s_control.server_ip }}:{{ k8s.api_server_port }}
      etcd:
        external:
          endpoints:
          {{ etcd.etcd_endpoint_list }}
      ---
      apiVersion: kubeadm.k8s.io/v1beta3
      kind: InitConfiguration
      bootstrapTokens:
        - token: "{{ k8s_control_node.cluster.token }}"
          ttl: "0"
      nodeRegistration:
        criSocket: /run/containerd/containerd.sock
      localAPIEndpoint:
        advertiseAddress: {{ k8s_control.server_ip }}
        bindPort: {{ k8s.api_server_port }}
      certificateKey: {{ k8s_control.certificate_key }}


  - path: /opt/bin/deploy-kube-system
    permissions: '0700'
    owner: root:root
    content: |
      #!/bin/bash -e

      if [[ -f "/home/cloud/success" ]]; then
      echo "Already provisioned!"
      exit 0
      fi

      if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then
        export PATH=$PATH:/opt/bin
      fi

      EXTERNAL_ETCD_NODES={{ etcd.unstacked_etcd }}
      EXTERNAL_CNI_PLUGIN={{ k8s.external.cni.plugin }}
      MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3
      crucial_cmd_attempts=1
      while true; do
        if (( "$crucial_cmd_attempts" > "$MAX_SETUP_CRUCIAL_CMD_ATTEMPTS" )); then
          echo "Error: kubeadm init failed!"
          exit 1
        fi
        retval=0
        set +e
        if [[ ${EXTERNAL_ETCD_NODES} == true ]]; then
          kubeadm init --config /etc/kubernetes/kubeadm-config.yaml --upload-certs
        else
          kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock
        fi
        retval=$?
        set -e
        if [ $retval -eq 0 ]; then
          break;
        fi
        crucial_cmd_attempts=$[$crucial_cmd_attempts + 1]
      done

      K8S_CONFIG_SCRIPTS_COPY_DIR=/tmp/k8sconfigscripts/

      if [[ $(systemctl is-active setup-kube-system) != "inactive" ]]; then
        echo "setup-kube-system is running!"
        exit 1
      fi
      if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then
        export PATH=$PATH:/opt/bin
      fi
      export KUBECONFIG=/etc/kubernetes/admin.conf

      mkdir -p /root/.kube
      cp -i /etc/kubernetes/admin.conf /root/.kube/config
      chown $(id -u):$(id -g) /root/.kube/config
      echo export PATH=\$PATH:/opt/bin >> /root/.bashrc

      if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then
        ### Network, dashboard configs available offline ###
        echo "Offline configs are available!"
        if [[ ${EXTERNAL_CNI_PLUGIN} == false ]]; then
          /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/network.yaml
        fi
        /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/dashboard.yaml
        rm -rf "${K8S_CONFIG_SCRIPTS_COPY_DIR}"
      else
        /opt/bin/kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(/opt/bin/kubectl version | base64 | tr -d '\n')"
        /opt/bin/kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta6/aio/deploy/recommended.yaml
      fi

      /opt/bin/kubectl create rolebinding admin-binding --role=admin --user=admin || true
      /opt/bin/kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=admin || true
      /opt/bin/kubectl create clusterrolebinding kubernetes-dashboard-ui --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard || true

      sudo touch /home/cloud/success
      echo "true" > /home/cloud/success

  {% if registry is defined %}
  - path: /opt/bin/setup-containerd
    permissions: '0755'
    owner: root:root
    content: |
      #!/bin/bash -e

      export registryConfig="\\        [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"{{registry.url.endpoint}}\"]\n \\         endpoint = [\"{{registry.url}}\"]"
      export registryCredentials="\\      [plugins.\"io.containerd.grpc.v1.cri\".registry.configs.\"{{registry.url.endpoint}}\".auth]\n\tusername = \"{{registry.username}}\" \n\tpassword = \"{{registry.password}}\" \n\tidentitytoken = \"{{registry.token}}\""

      echo "creating config file for containerd"
      containerd config default > /etc/containerd/config.toml
      sed  -i '/\[plugins."io.containerd.grpc.v1.cri".registry\]/a '"${registryCredentials}"'' /etc/containerd/config.toml
      sed  -i '/\[plugins."io.containerd.grpc.v1.cri".registry.mirrors\]/a '"${registryConfig}"'' /etc/containerd/config.toml

      echo "Restarting containerd service"
      systemctl daemon-reload
      systemctl restart containerd
  {% endif %}

  - path: /etc/systemd/system/deploy-kube-system.service
    permissions: '0755'
    owner: root:root
    content: |
      [Unit]
      Requires=containerd.service
      After=containerd.service

      [Service]
      Type=simple
      StartLimitInterval=0
      Restart=on-failure
      ExecStart=/opt/bin/deploy-kube-system

runcmd:
  - chown -R cloud:cloud /home/cloud/.ssh
  - containerd config default > /etc/containerd/config.toml
  - sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
  - until [ -f /opt/bin/setup-kube-system ]; do sleep 5; done
  - /opt/bin/setup-kube-system
  - until [ -f /etc/systemd/system/deploy-kube-system.service ]; do sleep 5; done
  - systemctl start deploy-kube-system