# 官方提供@yunTaoScripts 沙箱 🔥🔥

宿主机中运行容器，但是在宿主机中依然可以看到容器中的进程。通过沙箱实将容器隔离，宿主机就看不到容器进程了。 运行时分为高级别和低级别

高级别运行时
- docker
- podman
- container-d
- rkt
- cri-o
低级别运行时
- runc （默认）
- lxc
- gvisor
- kata

高低级别运行时区别

低级别运行时只能提供创建删除容器。
高级别运行时可以管理镜像，创建网络，卷。

# 非k8s集群环境使用

# 使用gvisor

# docker 下使用 gvisor

通过gvisor起容器，在宿主机是看不见容器进程的。

root@vms150:~# ll /usr/local/bin/   #所有人增加执行权限
-rwxr-xr-x  1 root root 26647799 Aug 20 07:15 containerd-shim-runsc-v1*
-rwxr-xr-x  1 root root 33632243 Aug 20 07:15 runsc*
root@vms150:/usr/local/bin# ./runsc install
2022/09/10 21:11:57 Added runtime "runsc" with arguments [] to "/etc/docker/daemon.json".
###更改运行时名称
root@vms150:~# cat /etc/docker/daemon.json 
{
    "runtimes": {
        "gvisor": {              ### 把runc改成gvisor
            "path": "/usr/local/bin/runsc"
        }
    }
}
root@vms150:~# dockerd --help | grep runtime
--default-runtime string        Default OCI runtime for containers (default "runc")
root@vms150:~# systemctl status docker.service 
● docker.service - Docker Application Container Engine
   Loaded: loaded (/lib/systemd/system/docker.service; enabled; vendor preset: enabled)
root@vms150:~# cat /lib/systemd/system/docker.service | grep -i execstart
ExecStart=/usr/bin/dockerd -H fd:// --default-runtime=gvisor --containerd=/run/containerd/containerd.sock   ##指定默认容器运行时，也可以直接在命令行中临时指定。
root@vms150:~# systemctl daemon-reload 
root@vms150:~# systemctl restart docker.service
root@vms150:~# docker run -dit --name gvisor --runtime gvisor nginx
19440e57953c68de1e34d34d8516bba102cc4a1cdd5b6062c7b2cbadf9fc353d
root@vms150:~# docker ps 
CONTAINER ID   IMAGE     COMMAND                  CREATED          STATUS         PORTS     NAMES
19440e57953c   nginx     "/docker-entrypoint.…"   10 seconds ago   Up 9 seconds   80/tcp    gvisor
root@vms150:~# ps -ef | grep nginx
root       2349   1254  0 21:02 pts/0    00:00:00 grep --color=auto nginx

# containerd下使用 gvisor

指定 gvisor 运行时，宿主机查看不到进程。

root@vms152:~# cat /etc/containerd/config.toml 
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]

        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
          base_runtime_spec = ""
          cni_conf_dir = ""
          cni_max_conf_num = 0
          container_annotations = []
          pod_annotations = []
          privileged_without_host_devices = false
          runtime_engine = ""
          runtime_path = ""
          runtime_root = ""
          runtime_type = "io.containerd.runc.v2"

          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
            SystemdCgroup = true
        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc] ###增加下面几行
          privileged_without_host_devices = false
          cni_conf_dir = ""
          cni_max_conf_num = 0
          base_runtime_spec = ""
          container_annotations = []
          runtime_type = "io.containerd.runsc.v1"
root@vms152:~# systemctl daemon-reload 
root@vms152:~# systemctl restart containerd.service
root@vms152:~# mv /etc/nerdctl/nerdctl.toml ~/   # 移除配置文件
root@vms152:~# export CONTAINERD_NAMESPACE=k8s.io #切换 nerdctl 在 containerd 的命名空间
root@vms152:~# nerdctl run -d --name gvisor --runtime runsc nginx                                                        
174fe2b6d8b9ee033b34a9e0711bb973c0267f77e979a7b00660918a15c53c8e
root@vms152:~# nerdctl ps
CONTAINER ID    IMAGE                             COMMAND                   CREATED           STATUS    PORTS    NAMES
174fe2b6d8b9    docker.io/library/nginx:latest    "/docker-entrypoint.…"    14 seconds ago    Up                 gvisor    
root@vms152:~# ps -ef | grep nginx
root       1523   1021  0 21:21 pts/0    00:00:00 grep --color=auto nginx

# 使用kata

在kvm虚拟机中创建一个容器，使用kata创建的已经不是容器，而是传统意义大的kvm虚拟机。

kvm 基于硬件辅助的虚拟化，必须要开启cpu虚拟化功能。

检查是否开启虚拟化（英特尔vmx，amd svm）

root@vms150:~# egrep 'vmx|svm' /proc/cpuinfo 
flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon nopl xtopology tsc_reliable nonstop_tsc cpuid pni pclmulqdq vmx ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch cpuid_fault invpcid_single pti ssbd ibrs ibpb stibp tpr_shadow vnmi ept vpid fsgsbase tsc_adjust bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt xsaveopt xsavec xgetbv1 xsaves arat md_clear flush_l1d arch_capabilities
flags		: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon nopl xtopology tsc_reliable nonstop_tsc cpuid pni pclmulqdq vmx ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch cpuid_fault invpcid_single pti ssbd ibrs ibpb stibp tpr_shadow vnmi ept vpid fsgsbase tsc_adjust bmi1 avx2 smep bmi2 invpcid rdseed adx smap clflushopt xsaveopt xsavec xgetbv1 xsaves arat md_clear flush_l1d arch_capabilities

# 在docker 下使用kata

vim /usr/lib/systemd/system/docker.service
ExecStart=/usr/bin/dockerd -D --add-runtime kata-runtime=/usr/bin/kata-runtime --default-runtime=kataruntime -H fd:// --containerd=/run/containerd/containerd.soc
root@vms152:~/kata-ubuntu# apt install ./*  ## 安装kata
root@vms152:~/kata-ubuntu# which kata-runtime 
/usr/bin/kata-runtime
root@vms150:/usr/local/bin# dockerd --help | grep add-runtime
      --add-runtime runtime                     Register an additional OCI compatible runtime (default [])
root@vms150:/usr/local/bin# systemctl status docker.service | grep lib
   Loaded: loaded (/lib/systemd/system/docker.service; enabled; vendor preset: enabled)
root@vms150:/usr/local/bin# grep -i execstart /lib/systemd/system/docker.service
ExecStart=/usr/bin/dockerd -H fd:// -D --add-runtime kata-my=/usr/bin/kata-runtime --containerd=/run/containerd/containerd.sock
root@vms150:/usr/local/bin# docker info | grep -i runtime -A 20   #增加了kata-my
 Runtimes: io.containerd.runtime.v1.linux kata-my runc runsc gvisor io.containerd.runc.v2
 Default Runtime: runc 
root@vms150:~/kata-ubuntu# docker run -dit --name kata --runtime kata-my nginx
2f73499dc924694f0844ac413e117c510456ee72d83ecee4f081e4e1bcc3c277
root@vms150:~/kata-ubuntu# ps aux | grep nginx   ##宿主机查看不到进程 
root       3738  0.0  0.0  13136  1004 pts/0    S+   22:12   0:00 grep --color=auto nginx
root@vms150:~/kata-ubuntu# docker inspect kata | grep -i runtime
            "Runtime": "kata-my",
            "CpuRealtimeRuntime": 0,

一定要开启硬件虚拟化

如果没有开启硬件虚拟化，kata运行不起来。

# 在containerd下使用kata

root@vms152:~/kata-ubuntu# find / -name "containerd-shim*"  ## runtime 写法参考
/usr/bin/containerd-shim-runc-v1
/usr/bin/containerd-shim-kata-v2
/usr/bin/containerd-shim
/usr/bin/containerd-shim-runc-v2
/usr/local/bin/containerd-shim-runsc-v1

root@vms152:~/kata-ubuntu# grep kata /etc/containerd/config.toml 
        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-runtime]
          runtime_type = "io.containerd.kata.v2"
root@vms152:~/kata-ubuntu# systemctl daemon-reload ;systemctl restart containerd.service 
root@vms152:~/kata-ubuntu# crictl info | grep -i runtime -A 20
"runtimes": {
        "kata-runtime": {
          "runtimeType": "io.containerd.kata.v2",
          "runtimePath": "",
          "runtimeEngine": "",
          "PodAnnotations": null,
          "ContainerAnnotations": null,
          "runtimeRoot": "",
          "options": null,
          "privileged_without_host_devices": false,
          "baseRuntimeSpec": "",
          "cniConfDir": "",
          "cniMaxConfNum": 0
        },
root@vms152:~/kata-ubuntu# nerdctl run -d --name kata1 --runtime kata-runtime nginx
aaf169f218ded1b24761bc361604645473c49ed95ac77ddddf2bb31819695c7f
root@vms152:~/kata-ubuntu# ps aux | grep nginx
root       3795  0.0  0.0  13136  1004 pts/0    S+   22:44   0:00 grep --color=auto nginx
root@vms152:~/kata-ubuntu# nerdctl ps
CONTAINER ID    IMAGE                             COMMAND                   CREATED           STATUS    PORTS    NAMES
aaf169f218de    docker.io/library/nginx:latest    "/docker-entrypoint.…"    11 seconds ago    Up                 kata1

# k8s集群下使用

针对k8s 集群开发的opa，gatekeeper

在 34 节点创建 nginx pod，默认使用runc，可以在宿主机中看到nginx进程

root@vms33:~# cat pod1.yaml 
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    run: pod1
  name: pod1
spec:
  nodeName: vms34.rhce.cc
  terminationGracePeriodSeconds: 0
  containers:
  - image: nginx
    imagePullPolicy: IfNotPresent
    name: pod1
    resources: {}
  dnsPolicy: ClusterFirst
  restartPolicy: Always
status: {}
root@vms33:~# kubectl get pod -A -owide
NAMESPACE        NAME                                       READY   STATUS    RESTARTS         AGE     IP               NODE     
netpolicy        pod1                                       1/1     Running   0                3s      10.244.194.144   vms34.rhce.cc   <none>           <none>
root@vms34:~# ps -ef | grep nginx
root       1550   1208  0 07:28 pts/0    00:00:00 grep --color=auto nginx
root@vms34:~# ps -ef | grep nginx
root       5525   5453  0 07:37 ?        00:00:00 nginx: master process nginx -g daemon off;
systemd+   5572   5525  0 07:37 ?        00:00:00 nginx: worker process
systemd+   5573   5525  0 07:37 ?        00:00:00 nginx: worker process
root       5662   1208  0 07:37 pts/0    00:00:00 grep --color=auto nginx

# 安装 gvisor 和 kata

安装gvisor

root@vms34:/usr/local/bin# chmod a+rx *
root@vms34:/usr/local/bin# ll
-rwxr-xr-x  1 root root 26647799 Aug 20 07:15 containerd-shim-runsc-v1*
-rwxr-xr-x  1 root root 33632243 Aug 20 07:15 runsc*

安装kata

root@vms34:~# tar -zxf kata-ubuntu.tar.gz
root@vms34:~# cd kata-ubuntu/
root@vms34:~/kata-ubuntu# apt install ./*

增加containerd 容器运行时

root@vms34:/etc/containerd# find / -name "containerd-shim*"
/usr/bin/containerd-shim-runc-v1
/usr/bin/containerd-shim-kata-v2
/usr/bin/containerd-shim
/usr/bin/containerd-shim-runc-v2
/usr/local/bin/containerd-shim-runsc-v1
root@vms34:~# vim /etc/containerd/config.toml
 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
          base_runtime_spec = ""
          cni_conf_dir = ""
          cni_max_conf_num = 0
          container_annotations = []
          pod_annotations = []
          privileged_without_host_devices = false
          runtime_engine = ""
          runtime_path = ""
          runtime_root = ""
          runtime_type = "io.containerd.runc.v2"
          
          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
            SystemdCgroup = true
        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc] # gvisor
          runtime_type = "io.containerd.runsc.v1"
        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-runtime]
          runtime_type = "io.containerd.kata.v2"                        # kata          
root@vms34:~# crictl info | grep -i runtime -A 20
"runtimes": {
        "kata-runtime": {
          "runtimeType": "io.containerd.kata.v2",
          "runtimePath": "",
          "runtimeEngine": "",
          "PodAnnotations": null,
          "ContainerAnnotations": null,
          "runtimeRoot": "",
          "options": null,
          "privileged_without_host_devices": false,
          "baseRuntimeSpec": "",
          "cniConfDir": "",
          "cniMaxConfNum": 0
        },
        "runc": {
          "runtimeType": "io.containerd.runc.v2",
          "runtimePath": "",
          "runtimeEngine": "",
          "PodAnnotations": [],
          "ContainerAnnotations": [],
          "runtimeRoot": "",
          "options": {
            "SystemdCgroup": true
          },
          "privileged_without_host_devices": false,
          "baseRuntimeSpec": "",
          "cniConfDir": "",
          "cniMaxConfNum": 0
        },
        "runsc": {
          "runtimeType": "io.containerd.runsc.v1",
          "runtimePath": "",
          "runtimeEngine": "",
          "PodAnnotations": null,
          "ContainerAnnotations": null,
          "runtimeRoot": "",
          "options": null,
          "privileged_without_host_devices": false,
          "baseRuntimeSpec": "",
          "cniConfDir": "",
          "cniMaxConfNum": 0
        }
      },

# pod 启动指定 Runtime Classs (opens new window)

# RuntimeClass is defined in the node.k8s.io API group
apiVersion: node.k8s.io/v1
kind: RuntimeClass
metadata:
  # The name the RuntimeClass will be referenced by.
  # RuntimeClass is a non-namespaced resource.
  name: myclass 
# The name of the corresponding CRI configuration
handler: runsc   ## 已经定义过的runtime

# 创建gvisor runtime class

root@vms33:~# kubectl apply -f runsc.yaml 
runtimeclass.node.k8s.io/myclass created
root@vms33:~# kubectl get runtimeclasses.node.k8s.io 
NAME      HANDLER   AGE
myclass   runsc     11s

启动pod并引用 gvisor runtime class，此时在34 节点宿主机已经看不到nginx进程。

root@vms33:~# vim pod1.yaml 
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    run: pod1
  name: pod1
spec:
  nodeName: vms34.rhce.cc
  terminationGracePeriodSeconds: 0
  runtimeClassName: myclass                   ##引用runtime
  containers:
  - image: nginx
    imagePullPolicy: IfNotPresent
    name: pod1
    resources: {}
  dnsPolicy: ClusterFirst
  restartPolicy: Always
status: {}
root@vms33:~# kubectl apply -f pod1.yaml 
pod/pod1 created
root@vms33:~# kubectl get pod -owide
NAME   READY   STATUS    RESTARTS   AGE   IP               NODE            NOMINATED NODE   READINESS GATES
pod1   1/1     Running   0          8s    10.244.194.145   vms34.rhce.cc   <none>           <none>
root@vms34:~# ps -ef | grep nginx
root      19945   1208  0 08:18 pts/0    00:00:00 grep --color=auto nginx

# 创建kata runtime class

root@vms33:~# vim kata.yaml 
# RuntimeClass is defined in the node.k8s.io API group
apiVersion: node.k8s.io/v1
kind: RuntimeClass
metadata:
  # The name the RuntimeClass will be referenced by.
  # RuntimeClass is a non-namespaced resource.
  name: myclass2
# The name of the corresponding CRI configuration
handler: kata-runtime
root@vms33:~# kubectl get runtimeclasses.node.k8s.io 
NAME       HANDLER        AGE
myclass    runsc          8m59s
myclass2   kata-runtime   3m25s

启动pod并引用 kata runtime class，此时在34 节点宿主机已经看不到nginx进程。

root@vms33:~# vim pod2.yaml 
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    run: pod2
  name: pod2
spec:
  nodeName: vms34.rhce.cc
  terminationGracePeriodSeconds: 0
  runtimeClassName: myclass2
  containers:
  - image: nginx
    imagePullPolicy: IfNotPresent
    name: pod2
    resources: {}
  dnsPolicy: ClusterFirst
  restartPolicy: Always
status: {}
root@vms33:~# kubectl apply -f pod2.yaml
root@vms33:~# kubectl get pod -owide
NAME   READY   STATUS    RESTARTS   AGE     IP               NODE           
pod2   1/1     Running   0          2m38s   10.244.194.146   vms34.rhce.cc   <none>           <none>

root@vms34:~# ps -ef | grep nginx
root      22589   1208  0 08:25 pts/0    00:00:00 grep --color=auto nginx

如果kata pod创建不起来怎么办？

grep "svm|vmx" /proc/cpuinfo  # 检查是否开启硬件虚拟化
modprobe kvm  # 开启内核模块
modprobe kvm_intel

← 快速链接

# 官方提供@yunTaoScripts 沙箱 🔥🔥

# 非k8s集群环境使用

# 使用gvisor

# docker 下 使用 gvisor

# containerd下 使用 gvisor

# 使用kata

# 在docker 下使用kata

# 在containerd下使用kata

# k8s集群下使用

# 安装 gvisor 和 kata

# pod 启动 指定 Runtime Classs (opens new window)

# 创建gvisor runtime class

# 创建kata runtime class

# docker 下使用 gvisor

# containerd下使用 gvisor

# pod 启动指定 Runtime Classs (opens new window)