# 官方提供@yunTaoScripts OPA 🔥🔥

loading

最小化服务漏洞,基于准入控制器开发的利用第三方提供工具来提升集群安全性。Open Policy Agent CNCF孵化的项目 (opens new window) ,并不只局限于k8s,还在ssh,sudo,docker等情况下使用。

# 非集群环境

没有opa规则时,docker命令不受任何限制。

# 安装docker opa 插件

OPA插件 官网 (opens new window)

docker plugin install openpolicyagent/opa-docker-authz-v2:0.4 opa-args="-policy-file /opa/policies/authz.rego" 
root@vms150:~# ls -l /var/lib/docker/plugins/
total 12
drwx------ 3 root root 4096 Sep 18 11:59 5b6f0234861720348a257703011c193d525fd389fc910b33ed2e213a24b3ecb2
drwxr-xr-x 4 root root 4096 Sep 18 11:59 storage
drwx------ 2 root root 4096 Sep 18 11:59 tmp

网速太慢可以压缩包直接装

tar -zxvf plug.tar.gz -C /var/lib/docker
systemctl restart docker
root@vms150:~# docker plugin ls
ID             NAME                                      DESCRIPTION                                     ENABLED
5b6f02348617   openpolicyagent/opa-docker-authz-v2:0.4   A policy-enabled authorization plugin for Do…   true

# 启用docker opa 插件

root@vms150:~# cat /etc/docker/daemon.json
{
    "runtimes": {
        "gvisor": {
            "path": "/usr/local/bin/runsc"
        },
        "runsc": {
            "path": "/usr/local/bin/runsc"
        }
    },
   "registry-mirrors": [ "https://frz7i079.mirror.aliyuncs.com"],
   "authorization-plugins": ["openpolicyagent/opa-docker-authz-v2:0.4"]
}
root@vms150:~# systemctl restart docker.service 
root@vms150:~# systemctl daemon-reload 

# 创建opa 规则

root@vms150:~# mkdir -p /etc/docker/policies
root@vms150:~# cat > /etc/docker/policies/authz.rego
package docker.authz
allow = true

# 拒绝所有 docker 命令

root@vms150:/etc/docker/policies# cat authz.rego 
package docker.authz
allow = false
root@vms150:/etc/docker/policies# docker ps
Error response from daemon: authorization denied by plugin openpolicyagent/opa-docker-authz-v2:0.4: request rejected by administrative policy

# 拒绝创建不设置 seccomp 规则的 容器

root@vms150:/etc/docker/policies# cat authz.rego 
package docker.authz
default allow = false
allow {
    not deny
}
deny {
    seccomp_unconfined
}
seccomp_unconfined {
input.Body.HostConfig.SecurityOpt[_] == "seccomp=unconfined"  ##列表 [_]
}
root@vms150:/etc/docker/policies# docker run -dit  --name=opa2 --security-opt seccomp=unconfined nginx
docker: Error response from daemon: authorization denied by plugin openpolicyagent/opa-docker-authz-v2:0.4: request rejected by administrative policy.
See 'docker run --help'.

规则查看

root@vms150:/etc/docker/policies# docker run -dit  --name=opa2 --security-opt seccomp=/root/allow.json nginx
b4197f2dd3185246c50717191464b9565704cc17f365b82d0b52c0622d534d34
root@vms150:/etc/docker/policies# docker inspect opa2 | grep seccomp -C 40
        "ExecIDs": null,
        "HostConfig": {
            "LogConfig": {
                "Type": "json-file",
                "Config": {}
            },
            "SecurityOpt": [
                "seccomp={\"defaultAction\":\"SCMP_ACT_ALLOW\"}"
            ],
  • input.Body是固定的,后面的是docker inspect 查看到的

# 限制不同登陆用户具备不同规则


root@vms150:~/.docker# cat /etc/docker/policies/authz.rego 
package docker.authz
default allow = false
# allow if the user is granted read/write access.
allow {    #规则1
    user_id := input.Headers["Authz-User"]
    user := users[user_id]
    not user.readOnly
}

allow {   # 规则2
    user_id := input.Headers["Authz-User"]
    users[user_id].readOnly 
    input.Method == "GET"  ##仅允许查询命令
}
users = {
    "bob": {"readOnly": true},
    "alice": {"readOnly": false},
}
root@vms150:~/.docker# cat config.json 
{
"HttpHeaders": {
    "Authz-User": "alice"}  #此时匹配规则1,所以创建成功
}
root@vms150:~/.docker# docker run -dit  --name=opa4 --security-opt seccomp=/root/allow.json nginx
1fee5375c74e799280c0796447ecdae54ea2e48553425bd8713cd8e8ad4c9e56

换成bob用户则启动失败

- 此时匹配规则2
root@vms150:~/.docker# cat config.json 
{
"HttpHeaders": {
    "Authz-User": "bob"}
}
root@vms150:~/.docker# docker run -dit  --name=opa3 --security-opt seccomp=/root/allow.json nginx
docker: Error response from daemon: authorization denied by plugin openpolicyagent/opa-docker-authz-v2:0.4: request rejected by administrative policy.
See 'docker run --help'.

# 集群环境

gatekeeper项目是Kubernetes基于OPA的实现。gatekeeper允许我们以Kubernetes本机方式使用OPA来实施所需的策略 安装gatekeeper之后,会生成constrainttemplates.templates.gatekeeper.sh 这个CRD然后利用这个CRD 可以创建template然后创建constraint

按照非集群模式安装opa,存在以下问题

  • 如果集群使用containerd,则无法安装opa插件。
  • 所有节点必须要同步opa 策略。
  • opa 单独为 k8s 开发了插件 gatekeeper

# 安装gatekeeeper

root@vms33:~/gatekeeper# nerdctl load -i gatekeeper-img.tar
root@vms33:~/gatekeeper# kubectl apply -f gatekeeper.yaml 
root@vms33:~/gatekeeper# kubectl get crd | grep gatekeeper
configs.config.gatekeeper.sh                          2022-09-18T05:31:23Z
constraintpodstatuses.status.gatekeeper.sh            2022-09-18T05:31:23Z
constrainttemplatepodstatuses.status.gatekeeper.sh    2022-09-18T05:31:23Z
constrainttemplates.templates.gatekeeper.sh           2022-09-18T05:31:23Z
providers.externaldata.gatekeeper.sh                  2022-09-18T05:31:23Z

# 创建规则

# 不允许使用特定镜像

  • 创建资源类型->限制镜像开头为docker-fakegoogle-gcr
root@vms33:~/gatekeeper# cat  gatekeeper-tmp.yaml 
apiVersion: templates.gatekeeper.sh/v1beta1
kind: ConstraintTemplate
metadata:
  name: blacklistimages
spec:
  crd:
    spec:
      names:
        kind: BlacklistImages
  targets:
  - rego: |
      package k8strustedimages ## 加载的规则可以随便写

      images {
        image := input.review.object.spec.containers[_].image
        not startswith(image, "docker-fake.io/")
        not startswith(image, "google-gcr-fake.com/")
        not startswith(image, "xxx/")  ##自行修改下镜像名字
      }

      violation[{"msg": msg}] {
        not images
        msg := "not trusted image!"
      }
    target: admission.k8s.gatekeeper.sh
root@vms33:~/gatekeeper# kubectl apply -f gatekeeper-tmp.yaml 
constrainttemplate.templates.gatekeeper.sh/blacklistimages created
root@vms33:~/gatekeeper# kubectl get crd | grep -i black
blacklistimages.constraints.gatekeeper.sh             2022-09-18T09:17:36Z
  • 创建资源对象
root@vms33:~/gatekeeper# cat gatekeeper-blacklist.yaml 
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: BlacklistImages
metadata:
  generation: 1
  managedFields:
  name: pod-trusted-images
  resourceVersion: "14449"
spec:
  match:
    kinds:
    - apiGroups:
      - ""
      kinds:
      - Pod
root@vms33:~/gatekeeper# kubectl apply -f gatekeeper-blacklist.yaml 
blacklistimages.constraints.gatekeeper.sh/pod-trusted-images created
root@vms33:~/gatekeeper# kubectl get BlacklistImages
NAME                 AGE
pod-trusted-images   6s
  • 创建pod 受到pod-trusted-images资源对象约束。
root@vms33:~/gatekeeper# cat opa-nginx.yaml 
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    run: opa-nginx
  name: opa-nginx
spec:
  nodeName: vms33.rhce.cc
  containers:
  - image: xxx/nginx
    imagePullPolicy: IfNotPresent
    name: opa-nginx
    resources: {}
  dnsPolicy: ClusterFirst
  restartPolicy: Always
status: {}
root@vms33:~/gatekeeper# kubectl apply -f opa-nginx.yaml 
Error from server ([pod-trusted-images] not trusted image!): error when creating "opa-nginx.yaml": admission webhook "validation.gatekeeper.sh" denied the request: [pod-trusted-images] not trusted image!

# 拒绝创建loadbalancer 类型的 Pod。

  • 创建资源类型
root@vms33:~/gatekeeper# cat aa-tmp.yaml 
apiVersion: templates.gatekeeper.sh/v1beta1
kind: ConstraintTemplate
metadata:
  name: lbtypesvcnotallowed
spec:
  crd:
    spec:
      names:
        kind: LBTypeSvcNotAllowed
  targets:
    - target: admission.k8s.gatekeeper.sh
      rego: |
        package kubernetes.admission
        violation[{"msg": msg}] {
                    input.review.kind.kind = "Service"
                    input.review.operation = "CREATE"
                    input.review.object.spec.type = "LoadBalancer"
                    msg := "LoadBalancer Services are not permitted"
        }
root@vms33:~/gatekeeper# kubectl apply -f aa-tmp.yaml 
constrainttemplate.templates.gatekeeper.sh/lbtypesvcnotallowed created
root@vms33:~/gatekeeper# kubectl get lbtypesvcnotallowed
No resources found
  • 创建资源对象
root@vms33:~/gatekeeper# cat bb-contraint.yaml 
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: LBTypeSvcNotAllowed
metadata:
  name: deny-lb-type-svc-dev-ns
spec:
  match:
    kinds:
      - apiGroups: [""]
        kinds: ["Service"]
    namespaces:
      - "admission"
root@vms33:~/gatekeeper# kubectl apply -f bb-contraint.yaml 
lbtypesvcnotallowed.constraints.gatekeeper.sh/deny-lb-type-svc-dev-ns created
root@vms33:~/gatekeeper# kubectl get lbtypesvcnotallowed
NAME                      AGE
deny-lb-type-svc-dev-ns   7s
  • 创建pod受到deny-lb-type-svc-dev-ns资源对象限制
root@vms33:~/gatekeeper# kubectl expose deployment dp1 --port 80 --type LoadBalancer --name prohibit-load
Error from server ([deny-lb-type-svc-dev-ns] LoadBalancer Services are not permitted): admission webhook "validation.gatekeeper.sh" denied the request: [deny-lb-type-svc-dev-ns] LoadBalancer Services are not permitted
root@vms33:~/gatekeeper# kubectl expose deployment dp1 --port 80 --type NodePort --name prohibit-load
service/prohibit-load exposed
最后修改时间: 12/31/2022, 12:00:03 PM