网站高防waf架构方案

一、背景

网站遭受攻击了，其中平均每5分钟的大概2000W+请求，一波下来人被折腾的够呛，梳理了大概情形后将其记录下来。项目目前架构采用GA->ALB->openresty->EKS模式，其中GA受Shield(通俗叫法为高防)保护，ALB受WAF保护。这里主要介绍aws waf及nginx-lua限并发操作，以下是我画的草图。

二、WAF操作

aws waf配置简单，这里忽略基本配置，重点讲解如何通过lambda对block ip进行自动写入黑名单操作。

2.1开启waf的logging

2.1.1创建传输流

2.1.2走默认下一步

2.2定时任务

CloudWatch–事件–规则–创建规则

2.3 配置Lambda

[root@localhost vhost]# cat auto.py
import json
import boto3
import time

SCOPE = 'REGIONAL'
WEB_ACL_NAME = 'fazzco-waf'
WEB_ACL_ID = 'd0bf5411-37d7-4993-938f-e7da66a4f9b4'
RULE_NAME = '5m-100'
DENY_IPV4_SET_NAME = 'BlockIPv4List'
DENY_IPV4_SET_ID = '6edaa8e8-da04-4024-95fd-e3baccd5a39c'

def lambda_handler(event, context):

    client = boto3.client('wafv2')

    ManagedIPV4 = client.get_rate_based_statement_managed_keys(
        Scope=SCOPE,
        WebACLName=WEB_ACL_NAME,
        WebACLId=WEB_ACL_ID,
        RuleName=RULE_NAME
    )

    if len(ManagedIPV4['ManagedKeysIPV4']['Addresses']) == 0:
        return {
            'statusCode': 200,
            'body': json.dumps('ManagedIPv4 is empty. Take no action')
        }

    # Once we get the ID, then we can list existed IP in block list.
    DenyIPv4Set = client.get_ip_set(
        Scope=SCOPE,
        Name=DENY_IPV4_SET_NAME,
        Id=DENY_IPV4_SET_ID
    )

    DENY_IPV4_SET_LOCK_TOKEN = DenyIPv4Set['LockToken']

    DenyIPv4Addrs = list(
        set(DenyIPv4Set['IPSet']['Addresses']) |
        set(ManagedIPV4['ManagedKeysIPV4']['Addresses']))

    if len(DenyIPv4Addrs) >= 10000:
        return {
        'statusCode': 500,
        'body': json.dumps('Error: DenyIPv4Addrs length is too long.')
    }

    response = client.update_ip_set(
        Scope=SCOPE,
        Name=DENY_IPV4_SET_NAME,
        Id=DENY_IPV4_SET_ID,
        Description='Last Update: '+time.ctime(),
        Addresses=DenyIPv4Addrs,
        LockToken=DENY_IPV4_SET_LOCK_TOKEN
    )

    return {
        'statusCode': 200,
        'body': json.dumps(response)
    }

备注需要修改的变量值：
WEB_ACL_NAME        waf的名称
WEB_ACL_ID                waf‘的ID值
RULE_NAME                使用哪个规则作为依据
DENY_IPV4_SET_NAME    IP池的名称
DENY_IPV4_SET_ID            IP池的id

2.4 权限配置

lambda权限管理

附加AWSWAFFullAccess权限

2.5 测试

2.6 架构模式

三、OpenResty配置

3.1 部署OpenResty

3.2 部署redis

3.3配置waf脚本

• waf-front.lua 当openresty处于最前端的waf脚本
• waf-elb.lua 当openresty处于elb后的waf脚本

-- waf-front.lua脚本
---
--- Generated by Luanalysis
--- Created by Yusin Xiang
--- DateTime: 2021/6/9 10:53
---
local ua = ngx.var.http_user_agent
local uri = ngx.var.request_uri
local url = ngx.var.host .. uri
local redis = require "resty.redis"
local red = redis.new()
local CCcount = 10
local CCseconds = 50
local Redisremote_ip = '127.0.0.1'
local RedisPORT = 6379
local blackseconds = 300

if ua == nil then
    ua = "unknown"
end

-- redis连接池设置
function close_redis(red)
    if not red then
        return
    end
    local pool_max_idle_time = 10000
    local pool_size = 100
    local ok, err = red:set_keepalive(pool_max_idle_time, pool_size)

    if not ok then
        ngx.log(ngx.ERR, "set redis keepalive error: ", err)
    end
end

-- 获取远程clientip
function getclientremote_ip()
    -- 当nginx处于最前端时
    local remote_ip = ngx.var.remote_addr

    if remote_ip == nil then
        remote_ip  = "unknown"
    end

    -- 匹配x_forwarded_for标头值是否带有','
    local index,to,err = ngx.re.find(remote_ip, ",", "jo")
    if index then
        ngx.log(ngx.ERR,'x_forwarded_for error: ',remote_ip)
        ngx.exit(403)
    end

    return remote_ip
end

function rule_ip(ip)
    -- 获取白名单匹配-192.168.1.5
    local white_flag = red:exists('white_ip')
    if white_flag == 1 then
        local res, err = red:lrange('white_ip', 0, 1)
        if res ~= nil then
            for i,v in ipairs(res) do
                --ngx.log(ngx.ERR, "ip:>>>> ", v)
                if remote_ip == v then
                    --close_redis(red)
                    ngx.log(ngx.ERR, "white ip ", v)
                    return 1
                end
            end
        else
            ngx.log(ngx.ERR, "white ip error+++++:  ", err)
        end
    end

    -- 黑名单处理
    local token = ip .. "." .. ngx.md5(url .. ua)
    local req = red:exists(token)
    if req == 0 then
        red:incr(token)
        red:expire(token,CCseconds)
        --close_redis(red)
        ngx.log(ngx.ERR, "blacklist++ ", remote_ip)
    else
        local count_number = tonumber(red:get(token))
        if count_number >= CCcount then
            local blackreq = red:exists("black." .. token)
            if blackreq  == 1 then
                --close_redis(red)
                ngx.log(ngx.ERR, "shield ip:", ip)
                ngx.exit(403)
            else
                red:set("black." .. token,1)
                red:expire("black." .. token,blackseconds)
                red:expire(token,blackseconds)
                --close_redis(red)
                ngx.exit(403)
            end
            return 2
        else
            red:incr(token)
            --close_redis(red)
            return 3
        end
    end
end

red:set_timeout(60)
local ok, err = red.connect(red, Redisremote_ip, RedisPORT)

-- 连接失败释放连接退出
if not ok then
    --close_redis(red)
    ngx.log(ngx.ERR, "Redis cannot connect ", err)
    --return ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
    return
end

-- 获取clientip
local remote_ip = getclientremote_ip()

local set_ip = rule_ip(remote_ip)
if set_ip then
    if set_ip == 1 then
        ngx.log(ngx.ERR,"return : ", '1')
    elseif set_ip == 2 then
        ngx.log(ngx.ERR,"return : ", '2')
    elseif set_ip == 3 then
        ngx.log(ngx.ERR,"return : ", '3')
    else
        ngx.log(ngx.ERR,"return : ", 'operation failed')
    end
end

local ok, err = red:close()
if not ok then
    ngx.log(ngx.ERR,"failed to close: ", err)
    return
else
    ngx.log(ngx.ERR,"success to close: ", err)
end

--- waf-elb.lua脚本
---
--- Generated by Luanalysis
--- Created by Yusin Xiang
--- DateTime: 2021/6/9 10:53
---
local ua = ngx.var.http_user_agent
local uri = ngx.var.request_uri
local url = ngx.var.host .. uri
local redis = require "resty.redis"
local red = redis.new()
local CCcount = 10
local CCseconds = 50
local Redisremote_ip = '172.20.20.206'
local RedisPORT = 6379
local blackseconds = 300
local private_network = '172.20'

if ua == nil then
    ua = "unknown"
end

-- redis连接池设置
--function close_redis(red)
--    if not red then
--        return
--    end
--    local pool_max_idle_time = 10000
--    local pool_size = 100
--    local ok, err = red:set_keepalive(pool_max_idle_time, pool_size)
--
--    if not ok then
--        ngx.log(ngx.ERR, "set redis keepalive error: ", err)
--    end
--end

-- 获取远程clientip
function getclientremote_ip()
    -- 当nginx处于alb下游时使用，当nginx处于最前端时参考waf-front.lua
    local remote_ip = ngx.req.get_headers()["x_forwarded_for"]

    --if remote_ip == nil then
    --    remote_ip  = ngx.var.remote_addr
    --end
    if remote_ip == nil then
        remote_ip  = "unknown"
    end

    -- 匹配x_forwarded_for标头值是否带有','
    local index,to,err = ngx.re.find(remote_ip, ",", "jo")
    if index then
        ngx.log(ngx.ERR,'x_forwarded_for error: ',remote_ip)
        ngx.exit(403)
    end

    return remote_ip
end

function rule_ip(ip)
    -- 匹配局域网ip网段，防止loadbance健康检查异常
    local from, to, err = ngx.re.find(ip, private_network, "jo")
    if from then
        return
    end

    -- 获取白名单匹配-192.168.1.5
    local white_flag = red:exists('white_ip')
    if white_flag == 1 then
        local res, err = red:lrange('white_ip', 0, 1)
        if res ~= nil then
            for i,v in ipairs(res) do
                --ngx.log(ngx.ERR, "ip:>>>> ", v)
                if remote_ip == v then
                    ngx.log(ngx.ERR, "white ip ", v)
                    return 1
                end
            end
        else
            ngx.log(ngx.ERR, "white ip error+++++:  ", err)
        end
    end

    -- 黑名单处理
    local token = ip .. "." .. ngx.md5(url .. ua)
    local req = red:exists(token)
    if req == 0 then
        red:incr(token)
        red:expire(token,CCseconds)
        ngx.log(ngx.ERR, "blacklist++ ", remote_ip)
    else
        local count_number = tonumber(red:get(token))
        if count_number >= CCcount then
            local blackreq = red:exists("black." .. token)
            if blackreq  == 1 then
                ngx.log(ngx.ERR, "shield ip:", ip)
                ngx.exit(403)
            else
                red:set("black." .. token,1)
                red:expire("black." .. token,blackseconds)
                red:expire(token,blackseconds)
                ngx.exit(403)
            end
            return 2
        else
            red:incr(token)
            return 3
        end
    end
end

-- 获取clientip
local remote_ip = getclientremote_ip()

red:set_timeout(60)
local ok, err = red.connect(red, Redisremote_ip, RedisPORT)

-- 连接失败释放连接退出
if not ok then
    ngx.log(ngx.ERR, "Redis cannot connect ", err)
    --return ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
    return
end

-- 获取clientip
local remote_ip = getclientremote_ip()

local set_ip = rule_ip(remote_ip)
if set_ip then
    if set_ip == 1 then
        ngx.log(ngx.ERR,"return : ", '1')
    elseif set_ip == 2 then
        ngx.log(ngx.ERR,"return : ", '2')
    elseif set_ip == 3 then
        ngx.log(ngx.ERR,"return : ", '3')
    else
        ngx.log(ngx.ERR,"return : ", 'operation failed')
    end
end

local ok, err = red:close()
if not ok then
    ngx.log(ngx.ERR,"failed to close: ", err)
    return
end

3.4 配置lua扩展

[root@ip-172-23-11-113 tmp]# wget https://raw.githubusercontent.com/openresty/lua-resty-redis/master/lib/resty/redis.lua
[root@ip-172-23-11-113 tmp]# mkdir /usr/local/openresty/lua-resty-redis/master/lib/resty/ -p
[root@ip-172-23-11-113 tmp]# cp redis.lua /usr/local/openresty/lua-resty-redis/master/lib/resty/

3.5 编辑nginx配置文件

[root@ip-172-23-11-113 conf]# vi nginx.conf
http区块下添加：
  lua_socket_pool_size 1024;
  lua_socket_keepalive_timeout 60s;
  lua_package_path "/usr/local/openresty/lua-resty-redis/master/lib/resty/redis.lua";

http区块下添加：
  access_by_lua_file /usr/local/openresty/nginx/conf/waf.lua;

四、总结

• 事发突然没有做好有效的防御措施，当攻击来临时手足无措
• 因预算原因，运维没有主动将安全纳入进整体架构之中
• 承接前面一条，临时将waf引入后并没有得到有效的防御，没能够及时梳理出aws waf特性，导致即使waf上线也无法做到有效防御
• lua限并发脚本初衷仅作为攻击时间前几分钟的一个防御，之后有aws waf作为主要防御点
• lua脚本属于临时编写，中间仅收录已知被攻击的报文类型，如其他攻击类型没有写入进去。并且没有在生产中验证过，如果要使用请慎用