#!/bin/sh # ---- # File: cilium-install.sh # Description: Tool to install k8s cilium test clusters using k3d or kind # Author: Sergio Talens-Oliag # Copyright: (c) 2023 Sergio Talens-Oliag # ---- set -e # Compute WORK_DIR SCRIPT="$(readlink -f "$0")" SCRIPT_DIR="$(dirname "$SCRIPT")" WORK_DIR_RELPATH=".." WORK_DIR="$(readlink -f "$SCRIPT_DIR/$WORK_DIR_RELPATH")" TMPL_DIR="$WORK_DIR/tmpl" YAML_DIR="$WORK_DIR/yaml" # --------- # VARIABLES # --------- GATEWAY_API_ENABLED="${GATEWAY_API_ENABLED:-false}" INGRESS_CONTROLLER_DEFAULT="${INGRESS_CONTROLLER_DEFAULT:-false}" INGRESS_CONTROLLER_ENABLED="${INGRESS_CONTROLLER_ENABLED:-false}" LOADBALANCER_MODE="shared" TUNNEL="vxlan" K3D_NETWORK_NAME="cilium" K3D_NET_PREFIX="172.30" K3D_CLUSTER_SUBNET_PREFIX="10.1" K3D_SERVICE_SUBNET_PREFIX="10.10" KIND_NETWORK_NAME="kind" KIND_NET_PREFIX="172.31" KIND_CLUSTER_SUBNET_PREFIX="10.2" KIND_SERVICE_SUBNET_PREFIX="10.20" NETWORK_TYPE="bridge" METALLB_ENABLED="true" METALLB_BASE_URL="https://raw.githubusercontent.com/metallb/metallb" METALLB_VERSION="v0.13.9" METALLB_DEPLOY_YAML="config/manifests/metallb-native.yaml" METALLB_YAML_URL="$METALLB_BASE_URL/$METALLB_VERSION/$METALLB_DEPLOY_YAML" METALLB_YAML="$YAML_DIR/metallb-native.yaml" NGINX_IC_ENABLED="true" NGINX_IC_BASE_URL="https://raw.githubusercontent.com/kubernetes/ingress-nginx" NGINX_IC_VERSION="controller-v1.7.0" NGINX_IC_DEPLOY_YAML="deploy/static/provider/cloud/deploy.yaml" NGINX_IC_YAML_URL="$NGINX_IC_BASE_URL/$NGINX_IC_VERSION/$NGINX_IC_DEPLOY_YAML" NGINX_IC_YAML="$YAML_DIR/ingress-nginx-deploy.yaml" # GOTMPLs TMPL_K3D_CONFIG_YAML="$TMPL_DIR/k3d-config.yaml" TMPL_KIND_CONFIG_YAML="$TMPL_DIR/kind-config.yaml" TMPL_IPPOOLS_YAML="$TMPL_DIR/ippools.yaml" TMPL_CILIUM_YAML="$TMPL_DIR/cilium.yaml" TMPL_METALLB_CRDS_YAML="$TMPL_DIR/metallb-crds.yaml" # Adjust variables based on other variables if [ "$METALLB_ENABLED" = "true" ]; then BGP_CONTROL_PLANE_ENABLED="false" else BGP_CONTROL_PLANE_ENABLED="true" fi # --------- # FUNCTIONS # --------- create_network() { NETWORK_ID="$( docker network inspect "$NETWORK_NAME" --format "{{.Id}}" 2>/dev/null )" || true if [ "$NETWORK_ID" ]; then echo "Using existing network '$NETWORK_NAME' with id '$NETWORK_ID'" else echo "Creating network '$NETWORK_NAME' in docker" docker network create \ --driver "$NETWORK_TYPE" \ --subnet "$NETWORK_SUBNET" \ --gateway "$NETWORK_GATEWAY" \ --ip-range "$NETWORK_IP_RANGE" \ "$NETWORK_NAME" fi } create_cluster() { echo "Creating $CTOOL cluster '$CNAME'" case "$CTOOL" in k3d) tmpl \ -v "cnum=$CNUM" \ -v "cname=$CNAME" \ -v "host_ip=$HOST_IP" \ -v "cluster_subnet=$CLUSTER_SUBNET" \ -v "service_subnet=$SERVICE_SUBNET" \ -v "work_dir=$WORK_DIR" \ "$TMPL_K3D_CONFIG_YAML" | k3d cluster create -c - ;; kind) tmpl \ -v "cnum=$CNUM" \ -v "cname=$CNAME" \ -v "host_ip=$HOST_IP" \ -v "cluster_subnet=$CLUSTER_SUBNET" \ -v "service_subnet=$SERVICE_SUBNET" \ -v "work_dir=$WORK_DIR" \ "$TMPL_KIND_CONFIG_YAML" | kind create cluster --config="-" ;; esac echo "Cluster '$CNAME' info" kubectl --context "$CTX" cluster-info } install_gateway_api_crds() { BASE_URL="https://raw.githubusercontent.com/kubernetes-sigs/gateway-api" BASE_URL="$BASE_URL/v0.5.1/config/crd" echo "Installing GatewayAPI CRDs" for crd_yaml in standard/gateway.networking.k8s.io_gatewayclasses.yaml \ standard/gateway.networking.k8s.io_gateways.yaml \ standard/gateway.networking.k8s.io_httproutes.yaml \ experimental/gateway.networking.k8s.io_referencegrants.yaml; do kubectl --context "$CTX" apply -f "$BASE_URL/$crd_yaml" done } cilium_status() { echo "Checking cilium status" cilium status --wait --context "$CTX" } master_node_ip() { # If we are not running kube-proxy the cilium Pods can't reach the api server # because the in-cluster service can't be reached, to fix the issue we use an # internal IP that the pods can reach, in this case we get the internal IP of # the master node container case "$CTOOL" in k3d) MASTER_NODE="node/$CTX-server-0";; kind) MASTER_NODE="node/$CNAME-control-plane";; *) echo "Unknown master node"; exit 1;; esac kubectl --context "$CTX" get "$MASTER_NODE" -o wide --no-headers | awk '{ print $6 }' } cilium_cli_install() { if [ "$GATEWAY_API_ENABLED" = "true" ]; then install_gateway_api_crds fi _xtra_args="" if [ "$CNUM" = "2" ]; then _xtra_args="--inherit-ca kind-cilium1" fi MASTER_NODE_IP="$(master_node_ip)" # shellcheck disable=SC2086 tmpl \ -v "master_node_ip=$MASTER_NODE_IP" \ -v "cnum=$CNUM" \ -v "cname=$CNAME" \ -v "bgp_control_plane_enabled=$BGP_CONTROL_PLANE_ENABLED" \ -v "gateway_api_enabled=$GATEWAY_API_ENABLED" \ -v "ingress_controller_default=$INGRESS_CONTROLLER_DEFAULT" \ -v "ingress_controller_enabled=$INGRESS_CONTROLLER_ENABLED" \ -v "loadbalancer_mode=$LOADBALANCER_MODE" \ -v "tunnel=$TUNNEL" \ "$TMPL_CILIUM_YAML" | cilium install --context "$CTX" --helm-values - $_xtra_args # Wait for the deployment cilium_status echo "Enabling hubble" cilium hubble enable --ui --context "$CTX" } cilium_helm_install() { if [ "$GATEWAY_API_ENABLED" = "true" ]; then install_gateway_api_crds fi helm repo add cilium https://helm.cilium.io/ >/dev/null || true # Copy the cilium-ca to the second cluster if [ "$CNUM" = "2" ]; then echo "Copying the 'cilium-ca' from '$CTOOL-cilium1' to '$CTX'" kubectl --context "$CTOOL-cilium1" -n kube-system get secrets/cilium-ca \ -o yaml | kubectl apply --context "$CTX" -f - fi MASTER_NODE_IP="$(master_node_ip)" # shellcheck disable=SC2086 tmpl \ -v "master_node_ip=$MASTER_NODE_IP" \ -v "cnum=$CNUM" \ -v "cname=$CNAME" \ -v "bgp_control_plane_enabled=$BGP_CONTROL_PLANE_ENABLED" \ -v "gateway_api_enabled=$GATEWAY_API_ENABLED" \ -v "ingress_controller_default=$INGRESS_CONTROLLER_DEFAULT" \ -v "ingress_controller_enabled=$INGRESS_CONTROLLER_ENABLED" \ -v "loadbalancer_mode=$LOADBALANCER_MODE" \ -v "tunnel=$TUNNEL" \ "$TMPL_CILIUM_YAML" | helm upgrade --install cilium cilium/cilium --version 1.13.1 \ --kube-context "$CTX" --namespace=kube-system --values=- } cilium_install(){ echo "Installing cilium in cluster '$CNAME'" cilium_helm_install cilium_status } lb_download_yaml() { [ -d "$YAML_DIR" ] || mkdir "$YAML_DIR" curl -fsSL -o "$METALLB_YAML" "$METALLB_YAML_URL" } lb_install() { if [ "$METALLB_ENABLED" = "true" ]; then if [ ! -f "$METALLB_YAML" ]; then lb_download_yaml fi echo "Installing metallb on kind cluster '$CNAME'" kubectl --context "$CTX" apply -f "$METALLB_YAML" echo "Waiting for metallb to be ready" kubectl --context "$CTX" rollout status deployment --timeout="120s" \ -n "metallb-system" "controller" echo "Configuring metallb" tmpl -v "lb_pool_range=$LB_POOL_RANGE" "$TMPL_METALLB_CRDS_YAML" | kubectl --context "$CTX" apply -f - elif [ "$BGP_CONTROL_PLANE_ENABLED" = "true" ]; then echo "Adding LB IPAM Pools" tmpl -v "lb_pool_cdir=$LB_POOL_CDIR" "$TMPL_IPPOOLS_YAML" | kubectl --context "$CTX" apply -f - fi } ingress_download_yaml() { [ -d "$YAML_DIR" ] || mkdir "$YAML_DIR" curl -fsSL -o "$NGINX_IC_YAML" "$NGINX_IC_YAML_URL" } ingress_install() { if [ "$NGINX_IC_ENABLED" = "true" ]; then if [ ! -f "$NGINX_IC_YAML" ]; then ingress_download_yaml fi echo "Installing nginx ingress controller on kind cluster '$CNAME'" kubectl --context "$CTX" apply -f "$NGINX_IC_YAML" echo "Waiting for the nginx controller to be ready" kubectl --context "$CTX" wait --namespace ingress-nginx \ --for=condition=ready pod \ --selector=app.kubernetes.io/component=controller \ --timeout=120s fi } mesh_install() { echo "Enabling cluster-mesh on cluster '$CNAME'" cilium clustermesh enable --context "$CTX" --service-type LoadBalancer echo "Checking cilium status on cluster '$CNAME'" cilium status --context "$CTX" --wait if [ "$CNUM" -eq "2" ]; then echo "Connecting cluster" cilium clustermesh connect --context "$CTOOL-cilium1" \ --destination-context "$CTOOL-cilium2" echo "Checking cilium status on cluster '$CNAME'" cilium status --context "$CTX" --wait fi } usage() { cat <