# 官方提供@yunTaoScripts 系统设置 Apparmor 🔥🔥

loading

从操作系统层面，提升系统安全性。 容器也是共享宿主机内核。

  • 物理机增加一个模块，pod和物理机同时总数加 1。
root@vms33:~# lsmod | wc -l
125
root@vms33:~# kubectl exec -it centos -- lsmod | wc -l
125
root@vms33:~# modprobe pcspkr     
root@vms33:~# kubectl exec -it centos -- lsmod | wc -l
126
root@vms33:~# lsmod | wc -l
126
  • 重启禁用以下两个模块
root@vms33:/etc/modprobe.d# cat xx.conf 
blacklist kvm
blacklist kvm_intel
  • 临时删除模块
root@vms33:/etc/modprobe.d# modprobe -r kvm_intel 
root@vms33:/etc/modprobe.d# kubectl exec -it centos -- lsmod | wc -l
123
root@vms33:/etc/modprobe.d# lsmod | wc -l
123

POLP 最小授权原则 （Principal of least privilege）

  • 启动模块一定要少
  • 授权按需最小授权
  • 权限分离，各司其职

# 非集群环境部署apparmor

类似selinux，centos 没有apparmor。设置进程可以访问哪些文件，不可以访问哪些文件。系统工具。

# 创建nginx应用

apt install nginx
mkdir -p /data/www/allow /data/www/deny
cp /var/www/html/index.nginx-debian.html /data/www/allow/index.html
cp /var/www/html/index.nginx-debian.html /data/www/deny/index.html
vim /data/www/allow/index.html 
vim /data/www/deny/index.html 

cd /etc/nginx/conf.d/
vim host.conf 
nginx -s reload
nginx -s reload

root@vms150:/etc/nginx/conf.d# cat host.conf 

server {
	listen 8080 default_server;
	root /data/www;
	index index.html index.htm index.nginx-debian.html;
	server_name _;
	location / {
		try_files $uri $uri/ =404;
	}
}

# 标准配置的文件

root@vms150:/etc/nginx/conf.d# ll /etc/apparmor.d/
-rw-r--r--  1 root root   125 Nov 23  2018 usr.bin.lxc-start
-rw-r--r--  1 root root  2857 Apr  7  2018 usr.bin.man
-rw-r--r--  1 root root 26245 Jul 10  2020 usr.lib.snapd.snap-confine.real
-rw-r--r--  1 root root  1550 Apr 24  2018 usr.sbin.rsyslogd  ##定义能访问到的文件
-rw-r--r--  1 root root  1353 Apr  1  2018 usr.sbin.tcpdump
  • 安装apparmor工具包，会多出一些规则，但是还没有nginx
root@vms150:/etc/nginx/conf.d# apt install apparmor-profiles apparmor-utils -y
  • 新增nginx规则
root@vms150:/etc/nginx/conf.d# aa-autodep nginx
Writing updated profile for /usr/sbin/nginx.
root@vms150:/etc/nginx/conf.d# ls /etc/apparmor.d/
sbin.dhclient   usr.lib.dovecot.config    usr.lib.dovecot.managesieve-login  usr.sbin.nginx
  • 标准配置文件
root@vms150:/etc/apparmor.d# cat usr.sbin.nginx 
# Last Modified: Tue Sep  6 11:09:42 2022
#include <tunables/global>

/usr/sbin/nginx flags=(complain) {                 ##标志位
  #include <abstractions/base>
  /lib/x86_64-linux-gnu/ld-*.so mr,
  /usr/sbin/nginx mr,    ##可读
}
root@vms150:/etc/apparmor.d# apparmor_parser -r usr.sbin.nginx #检查配置文件是否合规

标志位

  • 模式标志
    • enforce #不写flags=(complain) 代表强制模式
    • complain
  • 相对标志
    • chroot_relative
    • namespace_relative
  • 附加标志 2个值，各选一个
    • attch_disconnected
    • no_attch_disconnected 或者
    • chroot_attach
    • chroot_no_attach
  • 开启apparmor 强制模式后，nginx访问失败，进程重启失败。 开启后无权限访问nginx日志文件和配置文件
root@vms150:/etc/apparmor.d# aa-enforce nginx
Setting /usr/sbin/nginx to enforce mode.
root@vms150:/etc/apparmor.d# cat usr.sbin.nginx 
# Last Modified: Tue Sep  6 11:09:42 2022
#include <tunables/global>

/usr/sbin/nginx {
  #include <abstractions/base>

  /lib/x86_64-linux-gnu/ld-*.so mr,
  /usr/sbin/nginx mr,

}
  • 查看apparmor 限制，并解除限制
root@vms150:/etc/apparmor.d# aa-logprof 
Reading log entries from /var/log/syslog.
Updating AppArmor profiles in /etc/apparmor.d.
Enforce-mode changes:

Profile:    /usr/sbin/nginx
Capability: dac_override
Severity:   9

 [1 - #include <abstractions/lxc/container-base>]
  2 - #include <abstractions/lxc/start-container> 
  3 - capability dac_override, 
(A)llow / [(D)eny] / (I)gnore / Audi(t) / Abo(r)t / (F)inish
Adding #include <abstractions/lxc/container-base> to profile.
Deleted 2 previous matching profile entries.

= Changed Local Profiles =

The following local profiles were changed. Would you like to save them?

[1 - /usr/sbin/nginx]
(S)ave Changes / Save Selec(t)ed Profile / [(V)iew Changes] / View Changes b/w (C)lean profiles / Abo(r)t
Writing updated profile for /usr/sbin/nginx.
root@vms150:/etc/apparmor.d# cat usr.sbin.nginx 
# Last Modified: Tue Sep  6 11:35:33 2022
#include <tunables/global>

/usr/sbin/nginx {
  #include <abstractions/base>
  #include <abstractions/lxc/container-base>

}
  • 拒绝deny访问
root@vms150:/etc/apparmor.d# cat usr.sbin.nginx 
# Last Modified: Tue Sep  6 11:35:33 2022
#include <tunables/global>

/usr/sbin/nginx {
  #include <abstractions/base>
  #include <abstractions/lxc/container-base>
allow /data/www/allow/* r,
/data/www/allow/* r,  # 不写allow 或者 deny，默认allow
deny /data/www/deny/* r,  ## 同时写allow 和 deny， 默认deny

}
root@vms150:/etc/apparmor.d# apparmor_parser -r usr.sbin.nginx 
root@vms150:/etc/apparmor.d# nginx -s reload

# 集群下部署Apparmor

在k8s 集群下， 我们是 要对pod 的访问做限制，pod中运行什么进程我们也不知道，所以不适合用标准的配置文件。通过未关联的配置可以限制pod里的所有进程。

# 未关联的配置文件

  • 查看apparmor启用规则
root@vms34:~# apparmor_status 
apparmor module is loaded.
16 profiles are loaded.
16 profiles are in enforce mode.
   /sbin/dhclient
   /usr/bin/lxc-start
   /usr/bin/man
   /usr/lib/NetworkManager/nm-dhcp-client.action
   /usr/lib/NetworkManager/nm-dhcp-helper
   /usr/lib/connman/scripts/dhclient-script
   /usr/lib/snapd/snap-confine
   /usr/lib/snapd/snap-confine//mount-namespace-capture-helper
   /usr/sbin/tcpdump
   cri-containerd.apparmor.d
   lxc-container-default
   lxc-container-default-cgns
   lxc-container-default-with-mounting
   lxc-container-default-with-nesting
  • 创建配置并加载profile
root@vms34:/etc/apparmor.d# cat > aa.conf
#include <tunables/global>

profile profile1 {
  #include <abstractions/base>

}
root@vms34:/etc/apparmor.d# apparmor_parser -q aa.conf  ## 加载配置 文件 不发出警告
root@vms34:/etc/apparmor.d# apparmor_status | grep profile1
   profile1

apparmor_parser 用法

root@vms34:/etc/apparmor.d# apparmor_parser -h
-r, --replace		Replace apparmor definitions
-R, --remove		Remove apparmor definitions

# 创建pod并引用apparmor规则

root@vms33:~# cat pod2.yaml 
apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    run: pod2
  name: pod2
  annotations:
    container.apparmor.security.beta.kubernetes.io/pod2: localhost/profile1
                                                # 👆🏻容器名称 👆🏻pod所在节点 👆🏻节点apparmor规则
spec:
  nodeName: vms34.rhce.cc
  terminationGracePeriodSeconds: 0
  containers:
  - image: centos
    imagePullPolicy: IfNotPresent
    command: ["sh","-c","sleep 1d"]
    name: pod2
    resources: {}
  dnsPolicy: ClusterFirst
  restartPolicy: Always
status: {}
root@vms33:~# kubectl apply -f pod2.yaml 
pod/pod2 created
root@vms33:~# kubectl get pod
NAME   READY   STATUS             RESTARTS     AGE
pod2   0/1     CrashLoopBackOff   1 (4s ago)   5s
root@vms33:~# kubectl logs pod2
sh: /usr/bin/sleep: Permission denied
  • 增加apparmor 权限
root@vms34:~# cat /etc/apparmor.d/aa.conf 
#include <tunables/global>
profile profile1 {
  #include <abstractions/base>
  allow /usr/bin/sleep rwix,
}
root@vms34:~# apparmor_parser -r /etc/apparmor.d/aa.conf 
root@vms33:~# kubectl delete pod pod2
pod "pod2" deleted
root@vms33:~# kubectl apply -f pod2.yaml 
pod/pod2 created
root@vms33:~# kubectl logs pod2
root@vms33:~# kubectl get pod
NAME   READY   STATUS    RESTARTS   AGE
pod2   1/1     Running   0          11s
  • 执行命令
root@vms33:~# kubectl exec -it pod2 -- bash
bash: /root/.bashrc: Permission denied
  • 修改apparmor 规则 增加allow /root/.bashrc r,
root@vms34:~# cat /etc/apparmor.d/aa.conf 
#include <tunables/global>

profile profile1 {
  #include <abstractions/base>
  allow /usr/bin/sleep rwix,
  allow /root/.bashrc r,
}
  • 报错发生了变化
root@vms33:~# kubectl delete pod pod2
pod "pod2" deleted
root@vms33:~# kubectl apply -f pod2.yaml 
pod/pod2 created
root@vms33:~# kubectl exec -it pod2 -- bash
bash: /etc/bashrc: Permission denied
  • 修改apparmor 规则，增加allow /etc/bashrc r,
root@vms34:~# cat /etc/apparmor.d/aa.conf 
#include <tunables/global>

profile profile1 {
  #include <abstractions/base>
  allow /usr/bin/sleep rwix,
  allow /root/.bashrc r,
  allow /etc/bashrc r,
}
  • 此时sh 正常，但是ls 不正常。这样下去要一个一个授权很麻烦。
root@vms33:~# kubectl apply -f pod2.yaml
root@vms33:~# kubectl exec -it pod2 -- sh
root@vms33:~# kubectl exec -it pod2 -- sh                                            
sh: /usr/bin/ls: Permission denied
sh-4.#
  • 增加 #include <abstractions/bash> ,就可以 使用exec了。
root@vms34:~# cat /etc/apparmor.d/aa.conf 
#include <tunables/global>

profile profile1 {
  #include <abstractions/base>
  #include <abstractions/bash> 
  allow /usr/bin/sleep rwix,
  allow /usr/bin/ls rwix,
}
  • 此时 无法访问文件
root@vms33:~# kubectl exec -it  pod2 -- bash
[I have no name!@pod2 /]# ls
ls: cannot open directory '.': Permission denied
  • 增加 flags=(attach_disconnected) ，file。 root@vms34:~# cat /etc/apparmor.d/aa.conf #include <tunables/global> profile profile1 flags=(attach_disconnected){ #include <abstractions/base> #include <abstractions/bash> allow /usr/bin/sleep rwix, allow /usr/bin/ls rwix, file, }
  • 访问正常
root@vms33:~# kubectl exec -it  pod2 -- bash
[root@pod2 /]# ls
bin  dev  etc  home  lib  lib64  lost+found  media  mnt  opt  proc  root  run  sbin  srv  sys  tmp  usr  var
  • 限制特定目录
#include <tunables/global>
  
profile profile1 flags=(attach_disconnected){
  #include <abstractions/base>
  #include <abstractions/bash>
  allow /usr/bin/sleep rwix,
  allow /usr/bin/ls rwix,
  file,
  deny /tmp/ r,        ## 只限制 tmp 目录，不限制 子目录。
}

#include <tunables/global>
  
profile profile1 flags=(attach_disconnected){
  #include <abstractions/base>
  #include <abstractions/bash>
  allow /usr/bin/sleep rwix,
  allow /usr/bin/ls rwix,
  file,
  deny /tmp/* r,        ## 只限制 tmp 目录下第一层子目录。
}

#include <tunables/global>
  
profile profile1 flags=(attach_disconnected){
  #include <abstractions/base>
  #include <abstractions/bash>
  allow /usr/bin/sleep rwix,
  allow /usr/bin/ls rwix,
  file,
  deny /tmp/** r,        ## 限制 tmp 下所有子目录。
}
  • 换一个nginx镜像，pod 又会起不来，增加这行配置 #include <abstractions/lxc/container-base>
root@vms34:~# cat  /etc/apparmor.d/aa.conf 
#include <tunables/global>

profile profile1 flags=(attach_disconnected){
  #include <abstractions/base>
  #include <abstractions/bash>
  #include <abstractions/lxc/container-base>   ### 增加这行配置
  allow /usr/bin/sleep rwix,
  allow /usr/bin/ls rwix,
  file,
  deny /tmp/ r,
}
最后修改时间: 12/31/2022, 12:00:03 PM

←