Kubernetes: установка,настройка и эксплуатация кластера в инфраструктуре Openstack

 openstack server list --project k8s
+--------------------------------------+---------+--------+----------------------------------------+------------+------------+
| ID                                   | Name    | Status | Networks                               | Image      | Flavor     |
+--------------------------------------+---------+--------+----------------------------------------+------------+------------+
| 545358b8-1961-49d1-a8fb-468a29136e1e | kuber-4 | ACTIVE | kubernetes=172.27.113.16, 10.0.128.234 | ubuntu20v2 | m5.4xlarge |
| 9052d51f-e1f6-4b79-9063-8898454f4b3f | kuber-3 | ACTIVE | kubernetes=172.27.113.6, 10.0.128.233  | ubuntu20v2 | m5.4xlarge |
| fc6280ee-cff5-4094-915b-fffe646f814a | kuber-0 | ACTIVE | provider=10.0.128.230                  | ubuntu20v2 | m5.4xlarge |
| 6b3b045e-9724-42aa-afb5-fa70187e5585 | kuber-1 | ACTIVE | kubernetes=172.27.113.5, 10.0.128.231  | ubuntu20v2 | m5.4xlarge |
| 86a32b2e-e363-4ed4-b48b-501e012f932f | kuber-2 | ACTIVE | kubernetes=172.27.113.14, 10.0.128.232 | ubuntu20v2 | m5.4xlarge |
+--------------------------------------+---------+--------+----------------------------------------+------------+------------+

#!/bin/bash
cat <<EOF | sudo tee /etc/hosts
127.0.0.1   localhost
10.0.128.230    kuber-0
10.0.128.231    kuber-1
10.0.128.232    kuber-2
10.0.128.233    kuber-3
EOF

update-alternatives --config editor
dpkg-reconfigure tzdata

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

sudo sysctl --system
echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg

sudo apt-get update
sudo apt-get install -y kubelet kubeadm kubectl net-tools sockstat bridge-utils ifenslave

cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF

cat /etc/sysctl.d/99-kubernetes-cri.conf

cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables  = 1
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF

sudo sysctl --system
sudo mkdir -p /etc/containerd; containerd config default | sudo tee /etc/containerd/config.toml; sudo systemctl restart containerd
cat /etc/containerd/config.toml

sudo systemctl restart containerd

cat <<EOF | sudo tee /etc/modules-load.d/crio.conf
overlay
br_netfilter
EOF

cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables  = 1
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF

sudo sysctl --system

export VERSION=1.21
export OS='xUbuntu_20.04'
curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers.gpg add -
curl -L https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable:cri-o:$VERSION/$OS/Release.key | sudo apt-key --keyring /etc/apt/trusted.gpg.d/libcontainers-cri-o.gpg add -

cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /
EOF

cat <<EOF | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.list
deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/ /
EOF

# docker нужен для скачивания образов вс hub.docker.com

sudo apt-get update; sudo apt-get -y install cri-o cri-o-runc docker.io
sudo systemctl daemon-reload; sudo systemctl enable crio --now

#cat /etc/crio/crio.conf.d/02-cgroup-manager.conf; sleep 2

cat <<EOF | sudo tee /etc/docker/daemon.json
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2"
}
EOF

#sudo systemctl enable docker; sudo systemctl daemon-reload; sudo systemctl restart docker
echo -e "root soft nofile 102400\nroot hard nofile 102400" >> /etc/security/limits.conf
echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >> /etc/environment
# ssh authorization tune
echo -e "Match User ubuntu\n\tPasswordAuthentication yes" >> /etc/ssh/sshd_config
echo -e "\nHost kuber-*\n\tIdentityFile /home/ubuntu/.ssh/id_rsa" >> /etc/ssh/ssh_config

# Авторизация на hub.docker.com
docker login
sudo docker run hello-world

echo "Reboot start.."
sleep 3
reboot

ip a add 172.27.113.8/24 dev ens6
ip link set dev ens6 up
ip route add 172.27.113.0/24 dev ens6

FATA[0035] pulling image failed: rpc error: code = Unknown desc = Error reading manifest latest in quay.io/nginx: StatusCode: 404, <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final/...

gcloud container images list --project google-containers

sudo apt install docker.io

docker login
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded

# crictl pull nginx
DEBU[0000] PullImageRequest: &PullImageRequest{Image:&ImageSpec{Image:nginx,},Auth:nil,SandboxConfig:nil,}
DEBU[0008] PullImageResponse: &PullImageResponse{ImageRef:docker.io/library/nginx@sha256:2f1cd90e00fe2c991e18272bb35d6a8258eeb27785d121aa4cc1ae4235167cfd,}
Image is up to date for docker.io/library/nginx@sha256:2f1cd90e00fe2c991e18272bb35d6a8258eeb27785d121aa4cc1ae4235167cfd

#!/bin/bash
### 10.0.128.239 - IP address balancer of API, endpount LBaaS
export LBA="10.0.128.230"

function cluster_clear() {
sudo kubeadm reset --cri-socket /var/run/crio/crio.sock
#sudo apt purge kubectl kubeadm kubelet kubernetes-cni -y
#sudo apt autoremove
sudo rm -fr /etc/kubernetes/; sudo rm -fr ~/.kube/; sudo rm -fr /var/lib/etcd; sudo rm -rf /var/lib/cni/

# remove all running docker containers
docker rm -f `docker ps -a | grep "k8s_" | awk '{print $1}'`
echo Cluster remove complete..
}

function cluster_init() {
kubeadm init --control-plane-endpoint=${LBA} --apiserver-advertise-address=${LBA} --pod-network-cidr 10.85.0.0/16 --cri-socket /var/run/crio/crio.sock
}

#cluster_clear

sleep 5
cluster_init

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of control-plane nodes by copying certificate authorities
and service account keys on each node and then running the following as root:

  kubeadm join 10.0.128.230:6443 --token oewn3n.76uvps72a0xkqpr3 \
        --discovery-token-ca-cert-hash sha256:ab063820c72a614967e6c6c73e027fb21c329f2c9e5aff504b6a3fcd9e97162c \
        --control-plane

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 10.0.128.230:6443 --token oewn3n.76uvps72a0xkqpr3 \
        --discovery-token-ca-cert-hash sha256:ab063820c72a614967e6c6c73e027fb21c329f2c9e5aff504b6a3fcd9e97162c

kubectl proxy --address='10.0.128.230' --port 8080 --disable-filter=true

kubectl cluster-info
Kubernetes control plane is running at https://10.0.128.230:6443
CoreDNS is running at https://10.0.128.230:6443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy

kubectl cluster-info dump > cluster-info.dump

#!/bin/bash
LBA='10.0.128.114'
KT='h7eyg0.i77scifkhuyxl7l7'
KH='sha256:9ffb28f76e329995e42e06722484f6cbf54395521c40d92659783167460331e2'
CPI='--control-plane'
CRI='--cri-socket /var/run/crio/crio.sock'

function usage() {
   echo 'Usage: '$0' 1 (master) or 2 (worker)'
}

function pki_get() {
mkdir -p /etc/kubernetes/pki/etcd
if [[ $1 -eq 1 ]]; then
   scp -rp ubuntu@kuber-0:/etc/kubernetes/pki/ca.* /etc/kubernetes/pki
fi
scp -rp ubuntu@kuber-0:/etc/kubernetes/pki/sa.* /etc/kubernetes/pki
scp -rp ubuntu@kuber-0:/etc/kubernetes/pki/front* /etc/kubernetes/pki
scp -rp ubuntu@kuber-0:/etc/kubernetes/pki/etcd/ca.* /etc/kubernetes/pki/etcd
scp -rp ubuntu@kuber-0:/etc/kubernetes/pki/etcd/healthcheck* /etc/kubernetes/pki/etcd
}

case $1 in
        1)
        pki_get
        /usr/bin/kubeadm join ${LBA}:6443 --token $KT --discovery-token-ca-cert-hash $KH $CRI $CPI
        echo 'Master node joined to cluster'
        ;;
        2)
        pki_get
        /usr/bin/kubeadm join ${LBA}:6443 --token $KT --discovery-token-ca-cert-hash $KH $CRI
        echo 'Worker node joined to cluster'
        ;;
        *)
        usage
        ;;
esac

kubeadm join 10.0.128.183:6443 --token cnrhwi.hsiusc4b0hp93lfb \
        --discovery-token-ca-cert-hash sha256:75a3b276114b8147a6edac0fb669c6f4e14becb3002722ddc36f4cf799f0bc75 \
        --control-plane

 mkdir -p $HOME/.kube
 sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
 sudo chown $(id -u):$(id -g) $HOME/.kube/config

 kubectl get nodes

NAME      STATUS   ROLES                  AGE     VERSION
kuber-0   Ready    control-plane,master   99m     v1.21.2
kuber-1   Ready    control-plane,master   4m31s   v1.21.2

kubeadm join 10.0.128.183:6443 --token cnrhwi.hsiusc4b0hp93lfb \
        --discovery-token-ca-cert-hash sha256:75a3b276114b8147a6edac0fb669c6f4e14becb3002722ddc36f4cf799f0bc75

NAME      STATUS   ROLES                  AGE   VERSION
kuber-0   Ready    control-plane,master   22h   v1.21.2
kuber-1   Ready    control-plane,master   21h   v1.21.2
kuber-2   Ready    control-plane,master   21h   v1.21.2
kuber-3   Ready    <none>                 21h   v1.21.2
kuber-4   Ready    <none>                 16h   v1.21.2

kubeadm config print init-defaults

# kubeadm config print init-defaults
apiVersion: kubeadm.k8s.io/v1beta2
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 1.2.3.4
  bindPort: 6443
nodeRegistration:
  criSocket: /var/run/dockershim.sock
  name: node
  taints: null
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
  type: CoreDNS
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: k8s.gcr.io
kind: ClusterConfiguration
kubernetesVersion: 1.21.0
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
scheduler: {}

apt install etcd

cat << EOF > /etc/systemd/system/kubelet.service.d/20-etcd-service-manager.conf
[Service]
ExecStart=
#  Replace "systemd" with the cgroup driver of your container runtime. The default value in the kubelet is "cgroupfs".
ExecStart=/usr/bin/kubelet --address=127.0.0.1 --pod-manifest-path=/etc/kubernetes/manifests --cgroup-driver=systemd
Restart=always
EOF

systemctl restart kubelet

# cat kubeadm-conf-make
#!/bin/bash
# Update HOST0, HOST1, and HOST2 with the IPs or resolvable names of your hosts

export HOST0=172.27.113.14
export HOST1=172.27.113.12
export HOST2=172.27.113.13

# Create temp directories to store files that will end up on other hosts.
mkdir -p /tmp/${HOST0}/ /tmp/${HOST1}/ /tmp/${HOST2}/

ETCDHOSTS=(${HOST0} ${HOST1} ${HOST2})
NAMES=("infra0" "infra1" "infra2")

for i in "${!ETCDHOSTS[@]}"; do
HOST=${ETCDHOSTS[$i]}
NAME=${NAMES[$i]}
cat << EOF > /tmp/${HOST}/kubeadmcfg.yaml
apiVersion: "kubeadm.k8s.io/v1beta2"
kind: ClusterConfiguration
etcd:
    local:
        serverCertSANs:
        - "${HOST}"
        peerCertSANs:
        - "${HOST}"
        extraArgs:
            initial-cluster: ${NAMES[0]}=https://${ETCDHOSTS[0]}:2380,${NAMES[1]}=https://${ETCDHOSTS[1]}:2380,${NAMES[2]}=https://${ETCDHOSTS[2]}:2380
            initial-cluster-state: new
            name: ${NAME}
            listen-peer-urls: https://${HOST}:2380
            listen-client-urls: https://${HOST}:2379
            advertise-client-urls: https://${HOST}:2379
            initial-advertise-peer-urls: https://${HOST}:2380
EOF
done

#!/bin/bash
export HOST0=172.27.113.14
export HOST1=172.27.113.12
export HOST2=172.27.113.13

kubeadm init phase certs etcd-server --config=/tmp/${HOST2}/kubeadmcfg.yaml
kubeadm init phase certs etcd-peer --config=/tmp/${HOST2}/kubeadmcfg.yaml
kubeadm init phase certs etcd-healthcheck-client --config=/tmp/${HOST2}/kubeadmcfg.yaml
kubeadm init phase certs apiserver-etcd-client --config=/tmp/${HOST2}/kubeadmcfg.yaml
cp -R /etc/kubernetes/pki /tmp/${HOST2}/
# cleanup non-reusable certificates
find /etc/kubernetes/pki -not -name ca.crt -not -name ca.key -type f -delete

kubeadm init phase certs etcd-server --config=/tmp/${HOST1}/kubeadmcfg.yaml
kubeadm init phase certs etcd-peer --config=/tmp/${HOST1}/kubeadmcfg.yaml
kubeadm init phase certs etcd-healthcheck-client --config=/tmp/${HOST1}/kubeadmcfg.yaml
kubeadm init phase certs apiserver-etcd-client --config=/tmp/${HOST1}/kubeadmcfg.yaml
cp -R /etc/kubernetes/pki /tmp/${HOST1}/
find /etc/kubernetes/pki -not -name ca.crt -not -name ca.key -type f -delete

kubeadm init phase certs etcd-server --config=/tmp/${HOST0}/kubeadmcfg.yaml
kubeadm init phase certs etcd-peer --config=/tmp/${HOST0}/kubeadmcfg.yaml
kubeadm init phase certs etcd-healthcheck-client --config=/tmp/${HOST0}/kubeadmcfg.yaml
kubeadm init phase certs apiserver-etcd-client --config=/tmp/${HOST0}/kubeadmcfg.yaml
# No need to move the certs because they are for HOST0

# clean up certs that should not be copied off this host
find /tmp/${HOST2} -name ca.key -type f -delete
find /tmp/${HOST1} -name ca.key -type f -delete

#git clone  https://github.com/mikefarah/yq.git
export  VERSION=v4.2.0
export BINARY=yq_linux_amd64
wget https://github.com/mikefarah/yq/releases/download/v4.2.0/yq_linux_amd64.tar.gz -O - |  tar xz && mv ${BINARY} /usr/bin/yq

#!/bin/bash
set -euo pipefail

KUBEADM_CONFIG="${1-/tmp/kubeadm.yaml}"
echo "Printing to $KUBEADM_CONFIG"
echo "$KUBEADM_CONFIG is a directory!"

if [ -d "$KUBEADM_CONFIG" ]; then
 exit 1
fi

if [ ! -d $(dirname "$KUBEADM_CONFIG") ]; then
 echo "please create directory $(dirname $KUBEADM_CONFIG)"
 exit 1
fi

if [ ! $(which yq) ]; then
 echo "please install yq"
 exit 1
fi

if [ ! $(which kubeadm) ]; then
 echo "please install kubeadm"
 exit 1
fi

kubeadm config print init-defaults --component-configs=KubeletConfiguration > "$KUBEADM_CONFIG"

yq -i eval 'select(.nodeRegistration.criSocket) |= .nodeRegistration.criSocket = "unix:///var/run/crio/crio.sock"' "$KUBEADM_CONFIG"
yq -i eval 'select(di == 1) |= .cgroupDriver = "systemd"' "$KUBEADM_CONFIG"

root@kuber-0:/home/ubuntu# ./ka-cluster-go
[init] Using Kubernetes version: v1.21.2
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [kuber-0 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 172.27.113.14]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [kuber-0 localhost] and IPs [172.27.113.14 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [kuber-0 localhost] and IPs [172.27.113.14 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Starting the kubelet
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[kubelet-check] Initial timeout of 40s passed.

        Unfortunately, an error has occurred:
                timed out waiting for the condition

        This error is likely caused by:
                - The kubelet is not running
                - The kubelet is unhealthy due to a misconfiguration of the node in some way (required cgroups disabled)

        If you are on a systemd-powered system, you can try to troubleshoot the error with the following commands:
                - 'systemctl status kubelet'
                - 'journalctl -xeu kubelet'

        Additionally, a control plane component may have crashed or exited when started by the container runtime.
        To troubleshoot, list all containers using your preferred container runtimes CLI.

        Here is one example how you may list all Kubernetes containers running in cri-o/containerd using crictl:
                - 'crictl --runtime-endpoint /var/run/crio/crio.sock ps -a | grep kube | grep -v pause'
                Once you have found the failing container, you can inspect its logs with:
                - 'crictl --runtime-endpoint /var/run/crio/crio.sock logs CONTAINERID'

error execution phase wait-control-plane: couldn't initialize a Kubernetes cluster
To see the stack trace of this error execute with --v=5 or higher

kubeadm reset --cri-socket /var/run/crio/crio.sock

runtime-endpoint: "unix:///var/run/crio/crio.sock"
timeout: 0
debug: false

chown: changing ownership of '/var/lib/rabbitmq': Read-only file system
chown: changing ownership of '/data/configdb': Read-only file system
chown: changing ownership of '/data/db': Read-only file system

kubectl describe pod/carts-db-6c6c68b747-pbzwl -n sock-shop |  grep "Container ID"
chmod 666 /var/lib/containers/storage/overlay-containers/e2ed25b84587b31066f259e72a7180854ffae2723236ad6dcf0024396a31bee0

apt-get install -y containers-common libassuan-dev libdevmapper-dev libglib2.0-dev libc6-dev  libgpgme11-dev libgpg-error-dev libseccomp-dev libsystemd-dev libselinux1-dev pkg-config go-md2man cri-o-runc libudev-dev software-properties-common gcc make golang-go

kubectl proxy --address='10.0.128.230' --port 8080 --disable-filter=true

kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.3.1/aio/deploy/recommended.yaml

cat <<EOF | tee dashboard-adminuser.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kubernetes-dashboard
EOF

cat << EOF | tee dashboard-role-add.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kubernetes-dashboard
EOF

kubectl apply -f dashboard-adminuser.yaml
serviceaccount/admin-user created
kubectl apply -f dashboard-role-add.yaml
clusterrolebinding.rbac.authorization.k8s.io/admin-user created

# kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
Warning: policy/v1beta1 PodSecurityPolicy is deprecated in v1.21+, unavailable in v1.25+
podsecuritypolicy.policy/psp.flannel.unprivileged created
clusterrole.rbac.authorization.k8s.io/flannel created
clusterrolebinding.rbac.authorization.k8s.io/flannel created
serviceaccount/flannel created
configmap/kube-flannel-cfg created
daemonset.apps/kube-flannel-ds created

root@kuber-0:/home/ubuntu/k8s# ip a
7: flannel.1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN group default
    link/ether 72:fd:e8:7b:f3:a8 brd ff:ff:ff:ff:ff:ff
    inet 10.85.0.0/32 brd 10.85.0.0 scope global flannel.1
       valid_lft forever preferred_lft forever
    inet6 fe80::70fd:e8ff:fe7b:f3a8/64 scope link
       valid_lft forever preferred_lft forever

# ifconfig flannel.1
flannel.1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450
        inet 10.85.0.0  netmask 255.255.255.255  broadcast 10.85.0.0
        inet6 fe80::70fd:e8ff:fe7b:f3a8  prefixlen 64  scopeid 0x20<link>
        ether 72:fd:e8:7b:f3:a8  txqueuelen 0  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 7 overruns 0  carrier 0  collisions 0

kubectl --kubeconfig /etc/kubernetes/admin.conf get pods -n kubernetes-dashboard
NAME                                         READY   STATUS              RESTARTS   AGE
dashboard-metrics-scraper-856586f554-26k9c   1/1     Running             0          23h
kubernetes-dashboard-67484c44f6-wljxt        1/1     Running             0          23h
kubernetes-dashboard-78c79f97b4-qp5jp        0/1     ContainerCreating   0          16m

root@kuber-0:/home/ubuntu# kubectl get svc --all-namespaces
NAMESPACE              NAME                        TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                  AGE
default                kubernetes                  ClusterIP   10.96.0.1        <none>        443/TCP                  25h
kube-system            kube-dns                    ClusterIP   10.96.0.10       <none>        53/UDP,53/TCP,9153/TCP   25h
kubernetes-dashboard   dashboard-metrics-scraper   ClusterIP   10.101.166.245   <none>        8000/TCP                 24h
kubernetes-dashboard   kubernetes-dashboard        ClusterIP   10.104.138.52    <none>        443/TCP                  24h

# kubectl config view
apiVersion: v1
clusters:
- cluster:
    certificate-authority-data: DATA+OMITTED
    server: https://10.0.128.230:6443
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: kubernetes-admin
  name: kubernetes-admin@kubernetes
current-context: kubernetes-admin@kubernetes
kind: Config
preferences: {}
users:
- name: kubernetes-admin
  user:
    client-certificate-data: REDACTED
    client-key-data: REDACTED

http://10.0.128.230:8080/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard:/proxy/

http://10.0.128.230:8080/api/v1/namespaces/kubernetes-dashboard/services/http:kubernetes-dashboard:/proxy/#/namespace?namespace=default

kind    "Status"
apiVersion  "v1"
metadata    {}
status  "Failure"
message "error trying to reach service: dial tcp 10.85.0.15:8443: connect: no route to host"
reason  "ServiceUnavailable"
code    503

kubectl -n kubernetes-dashboard get services
# kubectl -n kubernetes-dashboard get services
NAME                        TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE
dashboard-metrics-scraper   ClusterIP   10.101.166.245   <none>        8000/TCP   24h
kubernetes-dashboard        ClusterIP   10.104.138.52    <none>        443/TCP    24h

kubectl -n kubernetes-dashboard edit service kubernetes-dashboard

type: NodePort                   ### clusterIP to NodePort

kubectl -n kubernetes-dashboard get services
NAME                        TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)         AGE
dashboard-metrics-scraper   ClusterIP   10.101.166.245   <none>        8000/TCP        24h
kubernetes-dashboard        NodePort    10.104.138.52    <none>        443:30583/TCP   24h

# lsof -i tcp:30583
COMMAND    PID USER   FD   TYPE DEVICE SIZE/OFF NODE NAME
kube-prox 2420 root   14u  IPv4 119116      0t0  TCP *:30583 (LISTEN)

kubectl -n kubernetes-dashboard get secret $(kubectl -n kubernetes-dashboard get sa/admin-user -o jsonpath="{.secrets[0].name}") -o go-template="{{.data.token | base64decode}}"

# kubectl get pod -n kubernetes-dashboard
NAME                                         READY   STATUS             RESTARTS   AGE
dashboard-metrics-scraper-856586f554-26k9c   1/1     Running            0          28h
kubernetes-dashboard-78c79f97b4-qp5jp        0/1     CrashLoopBackOff   11         4h29m

# kubectl describe pod kubernetes-dashboard-78c79f97b4-qp5jp -n kubernetes-dashboard
Name:         kubernetes-dashboard-78c79f97b4-qp5jp
Namespace:    kubernetes-dashboard
Priority:     0
Node:         kuber-4/172.27.113.16
Start Time:   Wed, 30 Jun 2021 12:08:23 +0300
Labels:       k8s-app=kubernetes-dashboard
              pod-template-hash=78c79f97b4
Annotations:  <none>
Status:       Running
IP:           10.85.4.82
IPs:
  IP:           10.85.4.82
Controlled By:  ReplicaSet/kubernetes-dashboard-78c79f97b4
Containers:
  kubernetes-dashboard:
    Container ID:  cri-o://e6645526b24a4c5eb46269ec3b102d95c73f3a2be218428b8ee50ff9eb182d33
    Image:         kubernetesui/dashboard:v2.2.0
    Image ID:      docker.io/kubernetesui/dashboard@sha256:148991563e374c83b75e8c51bca75f512d4f006ddc791e96a91f1c7420b60bd9
    Port:          8443/TCP
    Host Port:     0/TCP
    Args:
      --auto-generate-certificates
      --namespace=kubernetes-dashboard
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    2
      Started:      Wed, 30 Jun 2021 16:38:48 +0300
      Finished:     Wed, 30 Jun 2021 16:39:18 +0300
    Ready:          False
    Restart Count:  12
    Liveness:       http-get https://:8443/ delay=30s timeout=30s period=10s #success=1 #failure=3
    Environment:    <none>
    Mounts:
      /certs from kubernetes-dashboard-certs (rw)
      /tmp from tmp-volume (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from kube-api-access-r67w4 (ro)
Conditions:
  Type              Status
  Initialized       True
  Ready             False
  ContainersReady   False
  PodScheduled      True
Volumes:
  kubernetes-dashboard-certs:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  kubernetes-dashboard-certs
    Optional:    false
  tmp-volume:
    Type:       EmptyDir (a temporary directory that shares a pod's lifetime)
    Medium:
    SizeLimit:  <unset>
  kube-api-access-r67w4:
    Type:                    Projected (a volume that contains injected data from multiple sources)
    TokenExpirationSeconds:  3607
    ConfigMapName:           kube-root-ca.crt
    ConfigMapOptional:       <nil>
    DownwardAPI:             true
QoS Class:                   BestEffort
Node-Selectors:              kubernetes.io/os=linux
Tolerations:                 node-role.kubernetes.io/master:NoSchedule
                             node.kubernetes.io/not-ready:NoExecute op=Exists for 300s
                             node.kubernetes.io/unreachable:NoExecute op=Exists for 300s
Events:
  Type     Reason                  Age                     From     Message
  ----     ------                  ----                    ----     -------
  Warning  FailedCreatePodSandBox  49m (x1005 over 4h32m)  kubelet  (combined from similar events): Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_kubernetes-dashboard-78c79f97b4-qp5jp_kubernetes-dashboard_bffd1c5b-accd-453c-9545-65f4a66f5615_0(a500901ff7977d8c249164398943e67fb0670b59a2480db7539f75af12a8f5b3): failed to set bridge addr: "cni0" already has an IP address different from 10.85.4.1/24
  Warning  FailedCreatePodSandBox  49m                     kubelet  Failed to create pod sandbox: rpc error: code = Unknown desc = failed to create pod network sandbox k8s_kubernetes-dashboard-78c79f97b4-qp5jp_kubernetes-dashboard_bffd1c5b-accd-453c-9545-65f4a66f5615_0(c49c20b34dca3432cbd6eb562b6e05ecc2bbe120a82c7133c1e163cdbc37f2fb): open /run/flannel/subnet.env: no such file or directory
  Normal   Pulled                  48m                     kubelet  Successfully pulled image "kubernetesui/dashboard:v2.2.0" in 34.36994369s
  Normal   Pulled                  47m                     kubelet  Successfully pulled image "kubernetesui/dashboard:v2.2.0" in 48.124360993s
  Warning  Unhealthy               46m                     kubelet  Liveness probe failed: Get "https://10.85.4.82:8443/": dial tcp 10.85.4.82:8443: connect: connection refused
  Normal   Pulled                  46m                     kubelet  Successfully pulled image "kubernetesui/dashboard:v2.2.0" in 2.138998241s
  Normal   Pulling                 45m (x4 over 48m)       kubelet  Pulling image "kubernetesui/dashboard:v2.2.0"
  Normal   Created                 45m (x4 over 48m)       kubelet  Created container kubernetes-dashboard
  Normal   Started                 45m (x4 over 48m)       kubelet  Started container kubernetes-dashboard
  Normal   Pulled                  45m                     kubelet  Successfully pulled image "kubernetesui/dashboard:v2.2.0" in 2.169100957s
  Warning  BackOff                 3m56s (x177 over 46m)   kubelet  Back-off restarting failed container

curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
chmod 700 get_helm.sh
./get_helm.sh
helm repo add bitnami https://charts.bitnami.com/bitnami
helm search repo bitnami
helm repo update

# helm install bitnami/mysql --generate-name
NAME: mysql-1625151580
LAST DEPLOYED: Thu Jul  1 14:59:43 2021
NAMESPACE: default
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
** Please be patient while the chart is being deployed **

Tip:

  Watch the deployment status using the command: kubectl get pods -w --namespace default

Services:

  echo Primary: mysql-1625151580.default.svc.cluster.local:3306

Administrator credentials:

  echo Username: root
  echo Password : $(kubectl get secret --namespace default mysql-1625151580 -o jsonpath="{.data.mysql-root-password}" | base64 --decode)

To connect to your database:

  1. Run a pod that you can use as a client:

      kubectl run mysql-1625151580-client --rm --tty -i --restart='Never' --image  docker.io/bitnami/mysql:8.0.25-debian-10-r37 --namespace default --command -- bash

  2. To connect to primary service (read/write):

      mysql -h mysql-1625151580.default.svc.cluster.local -uroot -p my_database



To upgrade this helm chart:

  1. Obtain the password as described on the 'Administrator credentials' section and set the 'root.password' parameter as shown below:

      ROOT_PASSWORD=$(kubectl get secret --namespace default mysql-1625151580 -o jsonpath="{.data.mysql-root-password}" | base64 --decode)
      helm upgrade --namespace default mysql-1625151580 bitnami/mysql --set auth.rootPassword=$ROOT_PASSWORD

kubectl api-resources
# вывод команды дает большой список (у меня 58 строк), привожу несколько для примера
NAME                              SHORTNAMES   APIVERSION                             NAMESPACED   KIND
bindings                                       v1                                     true         Binding
componentstatuses                 cs           v1                                     false        ComponentStatus
nodes                             no           v1                                     false        Node
...
apiservices                                    apiregistration.k8s.io/v1              false        APIService
controllerrevisions                            apps/v1                                true         ControllerRevision
...
roles                                          rbac.authorization.k8s.io/v1           true         Role
priorityclasses                   pc           scheduling.k8s.io/v1                   false        PriorityClass
...
storageclasses                    sc           storage.k8s.io/v1                      false        StorageClass
volumeattachments                              storage.k8s.io/v1                      false        VolumeAttachment

$ kubectl get namespaces
NAME                   STATUS   AGE
default                Active   151m
kube-node-lease        Active   151m
kube-public            Active   151m
kube-system            Active   151m
kubernetes-dashboard   Active   94m

kubectl create namespace sock-shop

kubectl delete namespace sock-shop

kubectl run demo --image=nginx --port=9999 --labels app=demo
pod/demo created

kubectl run omegabi --image=dsagkomega/omegabi_angular --port=8888 --labels app=omegabi

kubectl get pod
NAME   READY   STATUS    RESTARTS   AGE
demo   1/1     Running   0          2m35s

kubectl get pod -n kube-system -w
NAME                              READY   STATUS    RESTARTS   AGE
coredns-558bd4d5db-9q5dl          1/1     Running   0          68m
coredns-558bd4d5db-bhjb4          1/1     Running   0          68m
etcd-kuber-0                      1/1     Running   0          69m
etcd-kuber-1                      1/1     Running   0          42m
etcd-kuber-2                      1/1     Running   0          37m
kube-apiserver-kuber-0            1/1     Running   0          69m
kube-apiserver-kuber-1            1/1     Running   0          42m
kube-apiserver-kuber-2            1/1     Running   0          37m
kube-controller-manager-kuber-0   1/1     Running   1          69m
kube-controller-manager-kuber-1   1/1     Running   0          42m
kube-controller-manager-kuber-2   1/1     Running   0          37m
kube-proxy-fbv4j                  1/1     Running   0          22m
kube-proxy-np29j                  1/1     Running   0          37m
kube-proxy-wr5fp                  1/1     Running   0          68m
kube-proxy-wv8c6                  1/1     Running   0          42m
kube-scheduler-kuber-0            1/1     Running   1          69m
kube-scheduler-kuber-1            1/1     Running   0          42m
kube-scheduler-kuber-2            1/1     Running   0          37m

kubectl delete pod hello
# или по селектору
kubectl delete all --selector app=demo

kubectl get deployments -n kube-system
NAME      READY   UP-TO-DATE   AVAILABLE   AGE
coredns   2/2     2            2           71m

kubectl describe deployments

kubectl apply -f application/deployment.yaml

# crictl images
IMAGE                                  TAG                 IMAGE ID            SIZE
docker.io/dsagkomega/omegabi_angular   latest              01c4c9b8e5164       477MB
k8s.gcr.io/coredns/coredns             v1.8.0              296a6d5035e2d       42.6MB
k8s.gcr.io/etcd                        3.4.13-0            0369cf4303ffd       255MB
k8s.gcr.io/kube-apiserver              v1.21.2             106ff58d43082       127MB
k8s.gcr.io/kube-controller-manager     v1.21.2             ae24db9aa2cc0       121MB
k8s.gcr.io/kube-proxy                  v1.21.2             a6ebd1c1ad981       133MB
k8s.gcr.io/kube-scheduler              v1.21.2             f917b8c8f55b7       51.9MB
k8s.gcr.io/pause                       3.4.1               0f8457a4c2eca       690kB
k8s.gcr.io/pause                       3.5                 ed210e3e4a5ba       690kB

# kubeadm config images list

kubectl api-resources | grep deployment
deployments                       deploy       apps/v1                                true         Deployment

apiVersion: apps/v1 # до версии 1.9.0 нужно использовать apps/v1beta2
kind: Deployment
metadata:
  name: nginx-deployment
spec:
  selector:
    matchLabels:
      app: nginx
  replicas: 2 # запускает 2 пода, созданных по шаблону
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx:1.14.2
        ports:
        - containerPort: 80

kubectl apply -f k8s/application/deployment.yaml
deployment.apps/nginx-deployment created

$ kubectl describe deployments
Name:                   nginx-deployment
Namespace:              default
CreationTimestamp:      Tue, 29 Jun 2021 14:58:22 +0300
Labels:                 <none>
Annotations:            deployment.kubernetes.io/revision: 1
Selector:               app=nginx
Replicas:               2 desired | 2 updated | 2 total | 0 available | 2 unavailable
StrategyType:           RollingUpdate
MinReadySeconds:        0
RollingUpdateStrategy:  25% max unavailable, 25% max surge
Pod Template:
  Labels:  app=nginx
  Containers:
   nginx:
    Image:        nginx:1.14.2
    Port:         80/TCP
    Host Port:    0/TCP
    Environment:  <none>
    Mounts:       <none>
  Volumes:        <none>
Conditions:
  Type           Status  Reason
  ----           ------  ------
  Available      False   MinimumReplicasUnavailable
  Progressing    False   ProgressDeadlineExceeded
OldReplicaSets:  <none>
NewReplicaSet:   nginx-deployment-66b6c48dd5 (2/2 replicas created)
Events:
  Type    Reason             Age   From                   Message
  ----    ------             ----  ----                   -------
  Normal  ScalingReplicaSet  12m   deployment-controller  Scaled up replica set nginx-deployment-66b6c48dd5 to 2

 containers:
  - image: nginx:1.14.2
    imagePullPolicy: IfNotPresent
    name: nginx
    ports:
    - containerPort: 80
      hostPort: 8888
      protocol: TCP

root@kuber-3:/home/ubuntu# sudo netstat -tulpn | grep 8888
tcp        0      0 0.0.0.0:8888            0.0.0.0:*               LISTEN      761/crio
tcp6       0      0 :::8888                 :::*                    LISTEN      761/crio

kubectl create namespace sock-shop

kubectl apply -n sock-shop -f "https://github.com/microservices-demo/microservices-demo/blob/master/deploy/kubernetes/complete-demo.yaml?raw=true"

# kubectl get pods -n sock-shop
NAME                            READY   STATUS              RESTARTS   AGE
carts-b4d4ffb5c-ztrgk           0/1     ContainerCreating   0          88s
carts-db-6c6c68b747-rmhtj       0/1     ContainerCreating   0          88s
catalogue-759cc6b86-62dpf       0/1     ContainerCreating   0          88s
catalogue-db-96f6f6b4c-wmwtt    0/1     ContainerCreating   0          88s
front-end-5c89db9f57-qlvm9      0/1     ContainerCreating   0          88s
orders-7664c64d75-44lr6         0/1     ContainerCreating   0          87s
orders-db-659949975f-5dd5l      0/1     ContainerCreating   0          87s
payment-7bcdbf45c9-v8xqv        0/1     ContainerCreating   0          87s
queue-master-5f6d6d4796-xvt5j   0/1     ContainerCreating   0          87s
rabbitmq-5bcbb547d7-tvcc5       0/2     ContainerCreating   0          87s
session-db-7cf97f8d4f-98jch     0/1     ContainerCreating   0          87s
shipping-7f7999ffb7-4fv7d       0/1     ContainerCreating   0          86s
user-68df64db9c-7mvbr           0/1     ContainerCreating   0          86s
user-db-6df7444fc-zkmqf         0/1     ContainerCreating   0          86s

kubectl describe svc front-end -n sock-shop
Name:                     front-end
Namespace:                sock-shop
Labels:                   name=front-end
Annotations:              prometheus.io/scrape: true
Selector:                 name=front-end
Type:                     NodePort
IP Family Policy:         SingleStack
IP Families:              IPv4
IP:                       10.110.238.210
IPs:                      10.110.238.210
Port:                     <unset>  80/TCP
TargetPort:               8079/TCP
NodePort:                 <unset>  30001/TCP
Endpoints:                10.85.3.160:8079
Session Affinity:         None
External Traffic Policy:  Cluster
Events:                   <none>

# kubectl get svc -n sock-shop
NAME           TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)             AGE
carts          ClusterIP   10.107.94.25     <none>        80/TCP              69m
carts-db       ClusterIP   10.108.113.150   <none>        27017/TCP           69m
catalogue      ClusterIP   10.99.209.178    <none>        80/TCP              69m
catalogue-db   ClusterIP   10.98.178.199    <none>        3306/TCP            69m
front-end      NodePort    10.110.238.210   <none>        80:30001/TCP        69m
orders         ClusterIP   10.98.139.44     <none>        80/TCP              69m
orders-db      ClusterIP   10.108.171.149   <none>        27017/TCP           69m
payment        ClusterIP   10.108.155.212   <none>        80/TCP              69m
queue-master   ClusterIP   10.96.95.92      <none>        80/TCP              69m
rabbitmq       ClusterIP   10.96.104.171    <none>        5672/TCP,9090/TCP   69m
session-db     ClusterIP   10.104.88.145    <none>        6379/TCP            69m
shipping       ClusterIP   10.98.104.222    <none>        80/TCP              69m
user           ClusterIP   10.109.61.179    <none>        80/TCP              69m
user-db        ClusterIP   10.106.92.76     <none>        27017/TCP           69m