官方提供@yunTaoScripts OPA 🔥🔥

最小化服务漏洞，基于准入控制器开发的利用第三方提供工具来提升集群安全性。Open Policy Agent CNCF孵化的项目 (opens new window) ，并不只局限于k8s，还在ssh，sudo，docker等情况下使用。

非集群环境

没有opa规则时，docker命令不受任何限制。

安装docker opa 插件

OPA插件 官网 (opens new window)

docker plugin install openpolicyagent/opa-docker-authz-v2:0.4 opa-args="-policy-file /opa/policies/authz.rego" 
root@vms150:~# ls -l /var/lib/docker/plugins/
total 12
drwx------ 3 root root 4096 Sep 18 11:59 5b6f0234861720348a257703011c193d525fd389fc910b33ed2e213a24b3ecb2
drwxr-xr-x 4 root root 4096 Sep 18 11:59 storage
drwx------ 2 root root 4096 Sep 18 11:59 tmp

网速太慢可以压缩包直接装

tar -zxvf plug.tar.gz -C /var/lib/docker
systemctl restart docker
root@vms150:~# docker plugin ls
ID             NAME                                      DESCRIPTION                                     ENABLED
5b6f02348617   openpolicyagent/opa-docker-authz-v2:0.4   A policy-enabled authorization plugin for Do…   true

启用docker opa 插件

root@vms150:~# cat /etc/docker/daemon.json
{
    "runtimes": {
        "gvisor": {
            "path": "/usr/local/bin/runsc"
        },
        "runsc": {
            "path": "/usr/local/bin/runsc"
        }
    },
   "registry-mirrors": [ "https://frz7i079.mirror.aliyuncs.com"],
   "authorization-plugins": ["openpolicyagent/opa-docker-authz-v2:0.4"]
}
root@vms150:~# systemctl restart docker.service 
root@vms150:~# systemctl daemon-reload 

创建opa 规则

root@vms150:~# mkdir -p /etc/docker/policies
root@vms150:~# cat > /etc/docker/policies/authz.rego
package docker.authz
allow = true

拒绝所有 docker 命令

root@vms150:/etc/docker/policies# cat authz.rego 
package docker.authz
allow = false
root@vms150:/etc/docker/policies# docker ps
Error response from daemon: authorization denied by plugin openpolicyagent/opa-docker-authz-v2:0.4: request rejected by administrative policy

拒绝创建不设置 seccomp 规则的 容器

root@vms150:/etc/docker/policies# cat authz.rego 
package docker.authz
default allow = false
allow {
    not deny
}
deny {
    seccomp_unconfined
}
seccomp_unconfined {
input.Body.HostConfig.SecurityOpt[_] == "seccomp=unconfined"  ##列表 [_]
}
root@vms150:/etc/docker/policies# docker run -dit  --name=opa2 --security-opt seccomp=unconfined nginx
docker: Error response from daemon: authorization denied by plugin openpolicyagent/opa-docker-authz-v2:0.4: request rejected by administrative policy.
See 'docker run --help'.

规则查看

root@vms150:/etc/docker/policies# docker run -dit  --name=opa2 --security-opt seccomp=/root/allow.json nginx
b4197f2dd3185246c50717191464b9565704cc17f365b82d0b52c0622d534d34
root@vms150:/etc/docker/policies# docker inspect opa2 | grep seccomp -C 40
        "ExecIDs": null,
        "HostConfig": {
            "LogConfig": {
                "Type": "json-file",
                "Config": {}
            },
            "SecurityOpt": [
                "seccomp={\"defaultAction\":\"SCMP_ACT_ALLOW\"}"
            ],

• input.Body是固定的,后面的是docker inspect 查看到的

限制不同登陆用户具备不同规则

root@vms150:~/.docker# cat /etc/docker/policies/authz.rego 
package docker.authz
default allow = false
# allow if the user is granted read/write access.
allow {    #规则1
    user_id := input.Headers["Authz-User"]
    user := users[user_id]
    not user.readOnly
}

allow {   # 规则2
    user_id := input.Headers["Authz-User"]
    users[user_id].readOnly 
    input.Method == "GET"  ##仅允许查询命令
}
users = {
    "bob": {"readOnly": true},
    "alice": {"readOnly": false},
}
root@vms150:~/.docker# cat config.json 
{
"HttpHeaders": {
    "Authz-User": "alice"}  #此时匹配规则1，所以创建成功
}
root@vms150:~/.docker# docker run -dit  --name=opa4 --security-opt seccomp=/root/allow.json nginx
1fee5375c74e799280c0796447ecdae54ea2e48553425bd8713cd8e8ad4c9e56

换成bob用户则启动失败

- 此时匹配规则2
root@vms150:~/.docker# cat config.json 
{
"HttpHeaders": {
    "Authz-User": "bob"}
}
root@vms150:~/.docker# docker run -dit  --name=opa3 --security-opt seccomp=/root/allow.json nginx
docker: Error response from daemon: authorization denied by plugin openpolicyagent/opa-docker-authz-v2:0.4: request rejected by administrative policy.
See 'docker run --help'.

集群环境

gatekeeper项目是Kubernetes基于OPA的实现。gatekeeper允许我们以Kubernetes本机方式使用OPA来实施所需的策略 安装gatekeeper之后，会生成constrainttemplates.templates.gatekeeper.sh 这个CRD然后利用这个CRD 可以创建template然后创建constraint

按照非集群模式安装opa，存在以下问题

• 如果集群使用containerd，则无法安装opa插件。
• 所有节点必须要同步opa 策略。
• opa 单独为 k8s 开发了插件 gatekeeper

安装gatekeeeper

root@vms33:~/gatekeeper# nerdctl load -i gatekeeper-img.tar
root@vms33:~/gatekeeper# kubectl apply -f gatekeeper.yaml 
root@vms33:~/gatekeeper# kubectl get crd | grep gatekeeper
configs.config.gatekeeper.sh                          2022-09-18T05:31:23Z
constraintpodstatuses.status.gatekeeper.sh            2022-09-18T05:31:23Z
constrainttemplatepodstatuses.status.gatekeeper.sh    2022-09-18T05:31:23Z
constrainttemplates.templates.gatekeeper.sh           2022-09-18T05:31:23Z
providers.externaldata.gatekeeper.sh                  2022-09-18T05:31:23Z

创建规则

不允许使用特定镜像

• 创建资源类型->限制镜像开头为docker-fake 和google-gcr

root@vms33:~/gatekeeper# cat  gatekeeper-tmp.yaml 
apiVersion: templates.gatekeeper.sh/v1beta1
kind: ConstraintTemplate
metadata:
  name: blacklistimages
spec:
  crd:
    spec:
      names:
        kind: BlacklistImages
  targets:
  - rego: |
      package k8strustedimages ## 加载的规则可以随便写

      images {
        image := input.review.object.spec.containers[_].image
        not startswith(image, "docker-fake.io/")
        not startswith(image, "google-gcr-fake.com/")
        not startswith(image, "xxx/")  ##自行修改下镜像名字
      }

      violation[{"msg": msg}] {
        not images
        msg := "not trusted image!"
      }
    target: admission.k8s.gatekeeper.sh
root@vms33:~/gatekeeper# kubectl apply -f gatekeeper-tmp.yaml 
constrainttemplate.templates.gatekeeper.sh/blacklistimages created
root@vms33:~/gatekeeper# kubectl get crd | grep -i black
blacklistimages.constraints.gatekeeper.sh             2022-09-18T09:17:36Z

• 创建资源对象

root@vms33:~/gatekeeper# cat gatekeeper-blacklist.yaml 
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: BlacklistImages
metadata:
  generation: 1
  managedFields:
  name: pod-trusted-images
  resourceVersion: "14449"
spec:
  match:
    kinds:
    - apiGroups:
      - ""
      kinds:
      - Pod
root@vms33:~/gatekeeper# kubectl apply -f gatekeeper-blacklist.yaml 
blacklistimages.constraints.gatekeeper.sh/pod-trusted-images created
root@vms33:~/gatekeeper# kubectl get BlacklistImages
NAME                 AGE
pod-trusted-images   6s

• 创建pod 受到pod-trusted-images资源对象约束。

root@vms33:~/gatekeeper# cat opa-nginx.yaml 
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    run: opa-nginx
  name: opa-nginx
spec:
  nodeName: vms33.rhce.cc
  containers:
  - image: xxx/nginx
    imagePullPolicy: IfNotPresent
    name: opa-nginx
    resources: {}
  dnsPolicy: ClusterFirst
  restartPolicy: Always
status: {}
root@vms33:~/gatekeeper# kubectl apply -f opa-nginx.yaml 
Error from server ([pod-trusted-images] not trusted image!): error when creating "opa-nginx.yaml": admission webhook "validation.gatekeeper.sh" denied the request: [pod-trusted-images] not trusted image!

拒绝创建loadbalancer 类型的 Pod。

• 创建资源类型

root@vms33:~/gatekeeper# cat aa-tmp.yaml 
apiVersion: templates.gatekeeper.sh/v1beta1
kind: ConstraintTemplate
metadata:
  name: lbtypesvcnotallowed
spec:
  crd:
    spec:
      names:
        kind: LBTypeSvcNotAllowed
  targets:
    - target: admission.k8s.gatekeeper.sh
      rego: |
        package kubernetes.admission
        violation[{"msg": msg}] {
                    input.review.kind.kind = "Service"
                    input.review.operation = "CREATE"
                    input.review.object.spec.type = "LoadBalancer"
                    msg := "LoadBalancer Services are not permitted"
        }
root@vms33:~/gatekeeper# kubectl apply -f aa-tmp.yaml 
constrainttemplate.templates.gatekeeper.sh/lbtypesvcnotallowed created
root@vms33:~/gatekeeper# kubectl get lbtypesvcnotallowed
No resources found

• 创建资源对象

root@vms33:~/gatekeeper# cat bb-contraint.yaml 
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: LBTypeSvcNotAllowed
metadata:
  name: deny-lb-type-svc-dev-ns
spec:
  match:
    kinds:
      - apiGroups: [""]
        kinds: ["Service"]
    namespaces:
      - "admission"
root@vms33:~/gatekeeper# kubectl apply -f bb-contraint.yaml 
lbtypesvcnotallowed.constraints.gatekeeper.sh/deny-lb-type-svc-dev-ns created
root@vms33:~/gatekeeper# kubectl get lbtypesvcnotallowed
NAME                      AGE
deny-lb-type-svc-dev-ns   7s

• 创建pod受到deny-lb-type-svc-dev-ns资源对象限制

root@vms33:~/gatekeeper# kubectl expose deployment dp1 --port 80 --type LoadBalancer --name prohibit-load
Error from server ([deny-lb-type-svc-dev-ns] LoadBalancer Services are not permitted): admission webhook "validation.gatekeeper.sh" denied the request: [deny-lb-type-svc-dev-ns] LoadBalancer Services are not permitted
root@vms33:~/gatekeeper# kubectl expose deployment dp1 --port 80 --type NodePort --name prohibit-load
service/prohibit-load exposed