Ubuntu 20.04 LTSのオンプレにKubernetes環境構築からnginx Pod稼働まで

# export http_proxy="http://192.168.0.158:3128"
# export https_proxy="http://192.168.0.158:3128"

# apt update
# apt install -y apt-transport-https ca-certificates curl software-properties-common

# curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -

# add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable"
# apt update

# apt install -y docker-ce

# docker -v
Docker version 20.10.12, build e91ed57

# systemctl status docker
● docker.service - Docker Application Container Engine
     Loaded: loaded (/lib/systemd/system/docker.service; enabled; vendor preset: enabled)
     Active: active (running) since Sat 2022-01-29 00:50:38 PST; 4min 44s ago

# curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg

# echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list
# apt update

# apt install -y kubelet kubeadm kubectl

# apt-mark hold kubelet kubeadm kubectl

# kubelet --version
Kubernetes v1.23.3
# kubeadm version
kubeadm version: &version.Info{Major:"1", Minor:"23", GitVersion:"v1.23.3", GitCommit:"816c97ab8cff8a1c72eccca1026f7820e93e0d25", GitTreeState:"clean", BuildDate:"2022-01-25T21:24:08Z", GoVersion:"go1.17.6", Compiler:"gc", Platform:"linux/amd64"}
# kubectl version
Client Version: version.Info{Major:"1", Minor:"23", GitVersion:"v1.23.3", GitCommit:"816c97ab8cff8a1c72eccca1026f7820e93e0d25", GitTreeState:"clean", BuildDate:"2022-01-25T21:25:17Z", GoVersion:"go1.17.6", Compiler:"gc", Platform:"linux/amd64"}
The connection to the server localhost:8080 was refused - did you specify the right host or port?

# mkdir -p /etc/systemd/system/docker.service.d
# echo -e "[Service]\nEnvironment=HTTP_PROXY=http://192.168.0.158:3128/ HTTPS_PROXY=http://192.168.0.158:3128/" | tee /etc/systemd/system/docker.service.d/http-proxy.conf

# systemctl daemon-reload
# systemctl restart docker

# vi /etc/docker/daemon.json

{
  "exec-opts": ["native.cgroupdriver=systemd"]
}

# systemctl daemon-reload
# systemctl restart docker

# vi /etc/fstab

#/swapfile     none     swap    sw    0    0

# systemctl --type swap
  UNIT          LOAD   ACTIVE SUB    DESCRIPTION
  swapfile.swap loaded active active /swapfile

# systemctl mask "swapfile.swap"
Created symlink /etc/systemd/system/swapfile.swap → /dev/null.

# systemctl --type swap
  UNIT          LOAD   ACTIVE SUB    DESCRIPTION
● swapfile.swap masked active active /swapfile

# reboot

# kubeadm init --node-name master --pod-network-cidr=10.244.0.0/16

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.12.200:6443 --token 95w8vq.89t27bukxp8dn6l7 \
        --discovery-token-ca-cert-hash sha256:e86f2ced6b3bd7a75e3b22cc54c4a6b47e2279b17f36192ecdc01904883ecffd

# export KUBECONFIG=/etc/kubernetes/admin.conf

# kubectl get nodes
NAME     STATUS     ROLES                  AGE    VERSION
master   NotReady   control-plane,master   6m9s   v1.23.3

# kubeadm join 192.168.12.200:6443 --node-name node1 --token 95w8vq.89t27bukxp8dn6l7 \
        --discovery-token-ca-cert-hash sha256:e86f2ced6b3bd7a75e3b22cc54c4a6b47e2279b17f36192ecdc01904883ecffd

# kubectl get nodes
NAME     STATUS     ROLES                  AGE   VERSION
master   NotReady   control-plane,master   46m   v1.23.3
node1    NotReady   <none>                 32s   v1.23.3

# sysctl -a | grep net.bridge.bridge-nf-call-iptables
net.bridge.bridge-nf-call-iptables = 1

# sysctl net.bridge.bridge-nf-call-iptables=1

# https_proxy="http://192.168.0.158:3128" no_proxy="192.168.12.200" kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+
podsecuritypolicy.policy/psp.flannel.unprivileged created
clusterrole.rbac.authorization.k8s.io/flannel created
clusterrolebinding.rbac.authorization.k8s.io/flannel created
serviceaccount/flannel created
configmap/kube-flannel-cfg created
daemonset.apps/kube-flannel-ds created

# kubectl get nodes
NAME     STATUS   ROLES                  AGE     VERSION
master   Ready    control-plane,master   6h33m   v1.23.3
node1    Ready    <none>                 5h29m   v1.23.3

# kubectl get pods
No resources found in default namespace.

# vi nginx-deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-deployment
spec:
  selector:
    matchLabels:
      app: nginx
  replicas: 3
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
        - name: nginx
          image: nginx:1.21.6-alpine
          ports:
            - containerPort: 80

# kubectl apply -f nginx-deployment.yaml
deployment.apps/nginx-deployment created

# kubectl get pods
NAME                                READY   STATUS              RESTARTS   AGE
nginx-deployment-5b7d486f5c-c8hzk   0/1     ContainerCreating   0          14s
nginx-deployment-5b7d486f5c-nms6k   0/1     ContainerCreating   0          14s
nginx-deployment-5b7d486f5c-q9mw8   0/1     ContainerCreating   0          14s

# kubectl get pods -o wide
NAME                                READY   STATUS    RESTARTS   AGE   IP           NODE    NOMINATED NODE   READINESS GATES
nginx-deployment-5b7d486f5c-c8hzk   1/1     Running   0          92s   10.244.1.4   node1   <none>           <none>
nginx-deployment-5b7d486f5c-nms6k   1/1     Running   0          92s   10.244.1.3   node1   <none>           <none>
nginx-deployment-5b7d486f5c-q9mw8   1/1     Running   0          92s   10.244.1.5   node1   <none>           <none>

# vi nginx-service.yaml

apiVersion: v1
kind: Service
metadata:
  name: nginx
spec:
  type: NodePort
  ports:
    - protocol: 'TCP'
      port: 8080
      targetPort: 80
      nodePort: 30080
  selector:
    app: nginx

# kubectl apply -f nginx-service.yaml

# kubectl get services -o wide
NAME         TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)          AGE   SELECTOR
kubernetes   ClusterIP   10.96.0.1        <none>        443/TCP          19h   <none>
nginx        NodePort    10.103.127.182   <none>        8080:30080/TCP   7s    app=nginx

# kubeadm init --node-name master --pod-network-cidr=10.244.0.0/16
W0125 23:40:54.844826    1779 version.go:103] could not fetch a Kubernetes version from the internet: unable to get URL "https://dl.k8s.io/release/stable-1.txt": Get "https://dl.k8s.io/release/stable-1.txt": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
W0125 23:40:54.844885    1779 version.go:104] falling back to the local client version: v1.23.3
[init] Using Kubernetes version: v1.23.3
[preflight] Running pre-flight checks
        [WARNING Swap]: swap is enabled; production deployments should disable swap unless testing the NodeSwap feature gate of the kubelet
        [WARNING Hostname]: hostname "master" could not be reached
        [WARNING Hostname]: hostname "master": lookup master on 127.0.0.53:53: server misbehaving
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'

# kubeadm config images pull
W0126 00:35:41.005320    1993 version.go:103] could not fetch a Kubernetes version from the internet: unable to get URL "https://dl.k8s.io/release/stable-1.txt": Get "https://dl.k8s.io/release/stable-1.txt": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
W0126 00:35:41.005376    1993 version.go:104] falling back to the local client version: v1.23.3
failed to pull image "k8s.gcr.io/kube-apiserver:v1.23.3": output: Error response from daemon: Get "https://k8s.gcr.io/v2/": dial tcp: lookup k8s.gcr.io: Temporary failure in name resolution
, error: exit status 1

# mkdir -p /etc/systemd/system/docker.service.d
# echo -e "[Service]\nEnvironment=HTTP_PROXY=http://192.168.0.158:3128/ HTTPS_PROXY=http://192.168.0.158:3128/" | sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf
[Service]
Environment=HTTP_PROXY=http://192.168.0.158:3128/ HTTPS_PROXY=http://192.168.0.158:3128/
# systemctl daemon-reload
# systemctl restart docker

# journalctl -xeu kubelet
Jan 27 17:20:49 ubuntu kubelet[342857]: E0127 17:20:49.359548  342857 server.go:302] "Failed to run kubelet" err="failed to run Kubelet: misconfiguration: kubelet cgroup driver: \"systemd\" is>
Jan 27 17:20:49 ubuntu systemd[1]: kubelet.service: Main process exited, code=exited, status=1/FAILURE

# vi /etc/docker/daemon.json

{
  "exec-opts": ["native.cgroupdriver=systemd"]
}

# systemctl daemon-reload
# systemctl restart docker

# kubeadm init --node-name master --pod-network-cidr=10.244.0.0/16
W0126 01:29:08.438000    6843 version.go:103] could not fetch a Kubernetes version from the internet: unable to get URL "https://dl.k8s.io/release/stable-1.txt": Get "https://dl.k8s.io/release/stable-1.txt": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
W0126 01:29:08.438145    6843 version.go:104] falling back to the local client version: v1.23.3
[init] Using Kubernetes version: v1.23.3
[preflight] Running pre-flight checks
        [WARNING Hostname]: hostname "master" could not be reached
        [WARNING Hostname]: hostname "master": lookup master on 127.0.0.53:53: server misbehaving
error execution phase preflight: [preflight] Some fatal errors occurred:
        [ERROR Port-6443]: Port 6443 is in use
        [ERROR Port-10259]: Port 10259 is in use

# kubeadm reset
# kubeadm init --node-name master --pod-network-cidr=10.244.0.0/16

# kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
The connection to the server localhost:8080 was refused - did you specify the right host or port?

# export KUBECONFIG=/etc/kubernetes/admin.conf

$ mkdir -p $HOME/.kube
$ sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
$ sudo chown $(id -u):$(id -g) $HOME/.kube/config

# https_proxy="http://192.168.0.158:3128" kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
Unable to connect to the server: Forbidden

# https_proxy="http://192.168.0.158:3128" no_proxy="192.168.12.200" kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+
podsecuritypolicy.policy/psp.flannel.unprivileged created
clusterrole.rbac.authorization.k8s.io/flannel created
clusterrolebinding.rbac.authorization.k8s.io/flannel created
serviceaccount/flannel created
configmap/kube-flannel-cfg created
daemonset.apps/kube-flannel-ds created

# kubeadm join 192.168.12.200:6443 --token 7bquah.q7ofwnr0obpeg44p \
>         --discovery-token-ca-cert-hash sha256:1c6e75a5f9743a6f766efb5d5dba1b82541c7a1614f3d93214f3b4b777756ba7
[preflight] Running pre-flight checks
error execution phase preflight: couldn't validate the identity of the API Server: could not find a JWS signature in the cluster-info ConfigMap for token ID "7bquah"
To see the stack trace of this error execute with --v=5 or higher

# kubeadm token list

# kubeadm token create --print-join-command
kubeadm join 192.168.12.200:6443 --token 558az8.uh111y2v4eme4dcj --discovery-token-ca-cert-hash sha256:1c6e75a5f9743a6f766efb5d5dba1b82541c7a1614f3d93214f3b4b777756ba7

# kubeadm join 192.168.12.200:6443 --token 558az8.uh111y2v4eme4dcj --discovery-token-ca-cert-hash sha256:1c6e75a5f9743a6f766efb5d5dba1b82541c7a1614f3d93214f3b4b777756ba7

# kubectl get pods -o wide
NAME                                READY   STATUS             RESTARTS   AGE     IP           NODE     NOMINATED NODE   READINESS GATES
nginx-deployment-84d98d877f-cs6dr   0/1     InvalidImageName   0          2m45s   10.244.1.3   node1    <none>           <none>
nginx-deployment-84d98d877f-m2pgj   0/1     InvalidImageName   0          2m45s   10.244.1.4   node1    <none>           <none>
nginx-deployment-84d98d877f-qxpvc   0/1     InvalidImageName   0          2m45s   10.244.1.2   node1    <none>           <none>

# kubectl delete deployment nginx-deployment

# kubectl apply -f nginx-deployment.yaml

# kubectl get services
NAME         TYPE           CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
kubernetes   ClusterIP      10.96.0.1       <none>        443/TCP          6h57m
nginx        LoadBalancer   10.107.14.190   <pending>     8080:32446/TCP   18s

# kubectl delete services nginx