春晚高并发论坛瘫痪，网络隧道转发，全站全球CDN加速升级

forum discourse
1

触发高并发的话题

触发防攻击系统

春晚电视直播软件话题，在春晚开始临近时产生高并发，触发了论坛系统防攻击系统，拦截了高并发ip地址，目前的网络隧道转发方案导致访问ip地址都是一致的，对现有方案做升级，让转发时保持原有ip地址。

image

方案测试

搭建服务器环境对方案做测试验证，再升级论坛网络隧道转发方案

服务器配置

image

网络工具

linux系统SoftEther VPN Server安装在两台服务器，一台为低配置低成本公网服务器，一台为高配置私网服务器
SoftEther 下载中心 (softether-download.com)

github加速

中国网络环境github访问慢做加速处理

安装softether-vpnserver

tar xzvf softether-vpnserver-v4.43-9799-beta-2023.08.31-linux-x64-64bit.tar.gz
cd vpnserver/
make
sudo mv ../vpnserver /usr/local
# 启动服务
sudo /usr/local/vpnserver/vpnserver start
# 使用桌面软件配置

安装softether-vpnclient

tar xzvf softether-vpnclient-v4.43-9799-beta-2023.08.31-linux-x64-64bit.tar.gz
cd vpnclient/
make
sudo mv ../vpnclient /usr/local
# 启动服务
sudo /usr/local/vpnclient/vpnclient start
# 命令行配置
/usr/local/vpnclient/vpncmd

配置命令

AccountDelete vpn
AccountCreate vpn
# 修改创建的账户
AccountSet
AccountGet
AccountPasswordSet vpn
AccountDisconnect vpn
AccountConnect vpn
AccountList
AccountStatusGet vpn
NicList
NicCreate vpn
NicDelete vpn

配置守护程序

sudo vi /usr/local/bin/softether-guard.sh
sudo chmod +x /usr/local/bin/softether-guard.sh
sudo vi /etc/systemd/system/softether-guard.service

[Unit]
Description=SoftEther VPN Guard
After=network.target

[Service]
ExecStart=/usr/local/bin/softether-guard.sh
Restart=always
User=root
StandardOutput=syslog
StandardError=syslog
SyslogIdentifier=softether-guard

[Install]
WantedBy=multi-user.target

sudo systemctl start softether-guard
# 使能重启自启动
sudo systemctl enable softether-guard
# 检查重启自启动
sudo systemctl is-enabled softether-guard
sudo systemctl disable softether-guard
sudo systemctl status softether-guard
sudo systemctl stop softether-guard

# 重新加载配置
systemctl daemon-reload
# 添加日志
sudo journalctl -u softether-guard
sudo journalctl --vacuum-time=1d
# 清除日志
sudo rm -rf /var/log/journal/*

中文系统守护脚本

#!/bin/bash

VPN_SERVER_PATH="/usr/local/vpnserver/vpnserver"
VPN_CLIENT_PATH="/usr/local/vpnclient/vpnclient"
VPN_CLIENT_CMD_PATH="/usr/local/vpnclient/vpncmd"
VPN_CONNECTION_NAME="vpn"

vpn_server_installed=false
vpn_client_installed=false

# 检查 SoftEther VPN Server 是否为可执行文件
if [ -x "$VPN_SERVER_PATH" ]; then
    vpn_server_installed=true
fi

# 检查 SoftEther VPN Client 是否为可执行文件
if [ -x "$VPN_CLIENT_PATH" ]; then
    vpn_client_installed=true
fi

# 检测 SoftEther VPN Server 是否在运行
check_vpn_server_status() {
    if pgrep -x vpnserver > /dev/null; then
        echo "SoftEther VPN Server is running."
    else
        echo "SoftEther VPN Server is not running. Starting it..."
        $VPN_SERVER_PATH start
    fi
}

# 检测 SoftEther VPN Client 是否在运行
check_vpn_client_status() {
    if pgrep -x vpnclient > /dev/null; then
        echo "SoftEther VPN Client is running."
    else
        echo "SoftEther VPN Client is not running. Starting it..."
        $VPN_CLIENT_PATH start
    fi
}

# 检测 VPN 连接是否已建立
check_vpn_connection() {
    if $VPN_CLIENT_CMD_PATH localhost /CLIENT /CMD AccountStatusGet $VPN_CONNECTION_NAME | grep -q "连接完成 (会话建立)"; then
        echo "VPN is connected."

        # 检查 VPN 接口是否有 IP 地址
        VPN_IP=$(ip addr show dev vpn_vpn | grep 'inet ' | awk '{print $2}')
        if [ -z "$VPN_IP" ]; then
            echo "VPN interface does not have an IP address. Requesting one..."
            sudo dhclient vpn_vpn
        else
            echo "VPN interface has the IP address: $VPN_IP"
        fi
    else
        echo "VPN is not connected. Connecting..."
        $VPN_CLIENT_CMD_PATH localhost /CLIENT /CMD AccountConnect $VPN_CONNECTION_NAME
    fi
}

while true; do
    if [ "$vpn_server_installed" = true ]; then
        echo "Checking VPN Server status..."
        check_vpn_server_status
    fi

    if [ "$vpn_client_installed" = true ]; then
        echo "Checking VPN Client status..."
        check_vpn_client_status
        check_vpn_connection
    fi

    sleep 60
done

英文系统守护脚本

#!/bin/bash

VPN_SERVER_PATH="/usr/local/vpnserver/vpnserver"
VPN_CLIENT_PATH="/usr/local/vpnclient/vpnclient"
VPN_CLIENT_CMD_PATH="/usr/local/vpnclient/vpncmd"
VPN_CONNECTION_NAME="vpn"

vpn_server_installed=false
vpn_client_installed=false

# 检查 SoftEther VPN Server 是否为可执行文件
if [ -x "$VPN_SERVER_PATH" ]; then
    vpn_server_installed=true
fi

# 检查 SoftEther VPN Client 是否为可执行文件
if [ -x "$VPN_CLIENT_PATH" ]; then
    vpn_client_installed=true
fi

# 检测 SoftEther VPN Server 是否在运行
check_vpn_server_status() {
    if pgrep -x vpnserver > /dev/null; then
        echo "SoftEther VPN Server is running."
    else
        echo "SoftEther VPN Server is not running. Starting it..."
        $VPN_SERVER_PATH start
    fi
}

# 检测 SoftEther VPN Client 是否在运行
check_vpn_client_status() {
    if pgrep -x vpnclient > /dev/null; then
        echo "SoftEther VPN Client is running."
    else
        echo "SoftEther VPN Client is not running. Starting it..."
        $VPN_CLIENT_PATH start
    fi
}

# 检测 VPN 连接是否已建立
check_vpn_connection() {
    if $VPN_CLIENT_CMD_PATH localhost /CLIENT /CMD AccountStatusGet $VPN_CONNECTION_NAME | grep -q "Session Established"; then
        echo "VPN is connected."

        # 检查 VPN 接口是否有 IP 地址
        VPN_IP=$(ip addr show dev vpn_vpn | grep 'inet ' | awk '{print $2}')
        if [ -z "$VPN_IP" ]; then
            echo "VPN interface does not have an IP address. Requesting one..."
            sudo dhclient vpn_vpn
        else
            echo "VPN interface has the IP address: $VPN_IP"
        fi
    else
        echo "VPN is not connected. Connecting..."
        $VPN_CLIENT_CMD_PATH localhost /CLIENT /CMD AccountConnect $VPN_CONNECTION_NAME
    fi
}

while true; do
    if [ "$vpn_server_installed" = true ]; then
        echo "Checking VPN Server status..."
        check_vpn_server_status
    fi

    if [ "$vpn_client_installed" = true ]; then
        echo "Checking VPN Client status..."
        check_vpn_client_status
        check_vpn_connection
    fi

    sleep 60
done

连接效果

image
image795×570 19.2 KB

论坛搭建

sudo git clone https://github.com/discourse/discourse_docker.git /var/discourse

论坛配置

论坛使用data和web分离方案，可以无感升级论坛，做到升级论坛不影响正常使用，支持邮件回复话题，涉及的配置文件有data.yml、web_only.yml、mail-receiver.yml，配置文件位于containers目录下

data.yml

# A container for all things Data, be sure to set a secret password for
# discourse account, SOME_SECRET is just an example
#

templates:
  - "templates/postgres.template.yml"
  - "templates/redis.template.yml"

# any extra arguments for Docker?
# docker_args:

params:
  db_default_text_search_config: "pg_catalog.english"

  ## Set db_shared_buffers to a max of 25% of the total memory.
  ## will be set automatically by bootstrap based on detected RAM, or you can override
  db_shared_buffers: "16384MB"

  ## can improve sorting performance, but adds memory usage per-connection
  db_work_mem: "128MB"

env:
  # ensure locale exists in container, you may need to install it
  LC_ALL: en_US.UTF-8
  LANG: en_US.UTF-8
  LANGUAGE: en_US.UTF-8

volumes:
  - volume:
        host: /var/discourse/shared/data
        guest: /shared
  - volume:
        host: /var/discourse/shared/data/log/var-log
        guest: /var/log

# TODO: SOME_SECRET to a password for the discourse user
hooks:
  after_postgres:
    - exec:
        stdin: |
          alter user discourse with password 'YWFlYTExMDk4YzhlYzJm';
        cmd: su - postgres -c 'psql discourse'

        raise_on_fail: false

web_only.yml
整体对无法访问的外服依赖库做了中国网络环境加速，xxx_key为相关私钥，替换为自己的就可以了

# IMPORTANT: SET A SECRET PASSWORD in Postgres for the Discourse User
# TODO: change SOME_SECRET in this template

templates:
  - "templates/web.template.yml"
  - "templates/web.china.template.yml"
  ## Uncomment the next line to enable the IPv6 listener
  #- "templates/web.ipv6.template.yml"
  #- "templates/web.ratelimited.template.yml"
  ## Uncomment these two lines if you wish to add Lets Encrypt (https)
  - "templates/web.ssl.template.yml"
  - "templates/web.letsencrypt.ssl.template.yml"
  - "templates/nginx_haproxy_ip_template.yml"

## which TCP/IP ports should this container expose?
## If you want Discourse to share a port with another webserver like Apache or nginx,
## see https://meta.discourse.org/t/17247 for details
expose:
  - "80:80"   # http
  - "443:443" # https

# Use 'links' key to link containers together, aka use Docker --link flag.
links:
  - link:
      name: data
      alias: data

# any extra arguments for Docker?
# docker_args:

params:
  ## Which Git revision should this container use? (default: tests-passed)
  #version: tests-passed

env:
  LC_ALL: en_US.UTF-8
  LANG: en_US.UTF-8
  LANGUAGE: en_US.UTF-8
  # DISCOURSE_DEFAULT_LOCALE: en

  ## How many concurrent web requests are supported? Depends on memory and CPU cores.
  ## will be set automatically by bootstrap based on detected CPUs, or you can override
  UNICORN_WORKERS: 16
  UNICORN_SIDEKIQ_MAX_RSS: 8192

  ## TODO: The domain name this Discourse instance will respond to
  DISCOURSE_HOSTNAME: www.zeppos.tech

  ## Uncomment if you want the container to be started with the same
  ## hostname (-h option) as specified above (default "$hostname-$config")
  #DOCKER_USE_HOSTNAME: true

  ## TODO: List of comma delimited emails that will be made admin and developer
  ## on initial signup example 'user1@example.com,user2@example.com'
  DISCOURSE_DEVELOPER_EMAILS: 'sobergou@gmail.com'

  ## TODO: The SMTP mail server used to validate new accounts and send notifications
  # SMTP ADDRESS, username, and password are required
  # WARNING the char '#' in SMTP password can cause problems!
  DISCOURSE_SMTP_ADDRESS: smtp.mailgun.org
  DISCOURSE_SMTP_PORT: 587
  DISCOURSE_SMTP_USER_NAME: postmaster@zeppos.tech
  DISCOURSE_SMTP_PASSWORD: "xxx_key"
  #DISCOURSE_SMTP_ENABLE_START_TLS: true           # (optional, default true)
  DISCOURSE_SMTP_DOMAIN: zeppos.tech
  DISCOURSE_NOTIFICATION_EMAIL: xahm@zeppos.tech

  ## If you added the Lets Encrypt template, uncomment below to get a free SSL certificate
  LETSENCRYPT_ACCOUNT_EMAIL: sobergou@gmail.com

  ## TODO: configure connectivity to the databases
  DISCOURSE_DB_SOCKET: ''
  #DISCOURSE_DB_USERNAME: discourse
  DISCOURSE_DB_PASSWORD: YWFlYTExMDk4YzhlYzJm
  DISCOURSE_DB_HOST: data
  DISCOURSE_REDIS_HOST: data

  ## The http or https CDN address for this Discourse instance (configured to pull)
  ## see https://meta.discourse.org/t/14857 for details
  #DISCOURSE_CDN_URL: https://discourse-cdn.example.com

  ## The maxmind geolocation IP address key for IP address lookup
  ## see https://meta.discourse.org/t/-/137387/23 for details
  DISCOURSE_MAXMIND_LICENSE_KEY: xxx_key

volumes:
  - volume:
      host: /var/discourse/shared/web-only
      guest: /shared
  - volume:
      host: /var/discourse/shared/web-only/log/var-log
      guest: /var/log

## Plugins go here
## see https://meta.discourse.org/t/19157 for details
hooks:
  after_code:
    - exec:
        cd: $home/plugins
        cmd:
          - git clone https://github.com/discourse/docker_manager.git
          - git clone https://github.com/discourse/discourse-graphviz.git
          - git clone https://github.com/discourse/discourse-math.git
          - git clone https://github.com/discourse/discourse-gamification.git
          - git clone https://github.com/discourse/discourse-translator.git
          - git clone https://github.com/discourse/discourse-cakeday.git
          - git clone https://github.com/discourse/discourse-saved-searches.git
          - git clone https://github.com/discourse/discourse-reactions.git
          #- git clone https://github.com/discourse/discourse-lti.git
          - git clone https://github.com/discourse/discourse-topic-voting.git
          #- git clone https://github.com/discourse/discourse-perspective-api.git
          - git clone https://github.com/discourse/discourse-user-notes.git
          - git clone https://github.com/discourse/discourse-microsoft-auth.git
          # - git clone https://github.com/discourse/discourse-yearly-review.git
          - git clone https://github.com/discourse/discourse-calendar.git
          - git clone https://github.com/discourse/discourse-encrypt.git
          #- git clone https://github.com/discourse/discourse-code-review.git
          - git clone https://github.com/discourse/discourse-policy.git
          #- git clone https://github.com/discourse/discourse-video.git
          #- git clone https://github.com/discourse/discourse-github.git
          - git clone https://github.com/discourse/discourse-docs.git
          # - git clone https://github.com/discourse/discourse-subscriptions.git
          - git clone https://github.com/discourse/discourse-adplugin.git
          # - git clone https://github.com/discourse/discourse-data-explorer.git
          - git clone https://github.com/discourse/discourse-ai.git
          - git clone https://github.com/discourse/discourse-follow.git
          - git clone https://github.com/discourse/discourse-whos-online

    - exec:
        cd: $home
        cmd:
          - gem sources --add https://gems.ruby-china.com/ --remove https://rubygems.org/
          - bundle config mirror.https://rubygems.org https://gems.ruby-china.com/
          - gem install tokenizers -v 0.3.3 -i /var/www/discourse/plugins/discourse-ai/gems/3.2.2 --no-document --ignore-dependencies --no-user-install
          - gem install tiktoken_ruby -v 0.0.5 -i /var/www/discourse/plugins/discourse-ai/gems/3.2.2 --no-document --ignore-dependencies --no-user-install
          - gem install aws-eventstream -v 1.2.0 -i /var/www/discourse/plugins/discourse-ai/gems/3.2.2 --no-document --ignore-dependencies --no-user-install
          - yarn config set registry https://registry.npmmirror.com/ --global
          - grep -rl "registry.yarnpkg.com" . | tee ./find_cache.txt | xargs sed -i "s/registry.yarnpkg.com/registry.npmmirror.com/g" && cat ./find_cache.txt && rm ./find_cache.txt

## Remember, this is YAML syntax - you can only have one block with a name
run:
  - exec: echo "Beginning of custom commands"

  ## If you want to configure password login for root, uncomment and change:
  ## Use only one of the following lines:
  #- exec: /usr/sbin/usermod -p 'PASSWORD_HASH' root
  #- exec: /usr/sbin/usermod -p "$(mkpasswd -m sha-256 'RAW_PASSWORD')" root

  ## If you want to authorized additional users, uncomment and change:
  #- exec: ssh-import-id username
  #- exec: ssh-import-id anotherusername

  - exec: echo "End of custom commands"
  - exec: awk -F\# '{print $1;}' ~/.ssh/authorized_keys | awk 'BEGIN { print "Authorized SSH keys for this container:"; } NF>=2 {print $NF;}'

mail-receiver.yml
邮箱回复话题配置，xxx_key为相关私钥，替换为自己的就可以了

## this is the incoming mail receiver container template
##
## After making changes to this file, you MUST rebuild
## /var/discourse/launcher rebuild mail-receiver
##
## BE *VERY* CAREFUL WHEN EDITING!
## YAML FILES ARE SUPER SUPER SENSITIVE TO MISTAKES IN WHITESPACE OR ALIGNMENT!
## visit http://www.yamllint.com/ to validate this file as needed

base_image: discourse/mail-receiver:release
update_pups: false

expose:
  - "25:25"   # SMTP

env:
  LC_ALL: en_US.UTF-8
  LANG: en_US.UTF-8
  LANGUAGE: en_US.UTF-8

  ## Where e-mail to your forum should be sent.  In general, it's perfectly fine
  ## to use the same domain as the forum itself here.
  MAIL_DOMAIN: zeppos.tech
# uncomment these (and the volume below!) to support TLS
  POSTCONF_smtpd_tls_key_file:  /letsencrypt/www.zeppos.tech/www.zeppos.tech.key
  POSTCONF_smtpd_tls_cert_file:  /letsencrypt/www.zeppos.tech/fullchain.cer
  POSTCONF_smtpd_tls_security_level: may


  ## The URL of the mail processing endpoint of your Discourse forum.
  ## This is simply your forum's base URL, with `/admin/email/handle_mail`
  ## appended.  Be careful if you're running a subfolder setup -- in that case,
  ## the URL needs to have the subfolder included!
  DISCOURSE_MAIL_ENDPOINT: 'https://www.zeppos.tech/admin/email/handle_mail'

  ## The master API key of your Discourse forum.  You can get this from
  ## the "API" tab of your admin panel.
  DISCOURSE_API_KEY: xxx_key

  ## The username to use for processing incoming e-mail.  Unless you have
  ## renamed the `system` user, you should leave this as-is.
  DISCOURSE_API_USERNAME: system

volumes:
  - volume:
      host: /var/discourse/shared/mail-receiver/postfix-spool
      guest: /var/spool/postfix
# uncomment to support TLS
  - volume:
      host: /var/discourse/shared/web-only/letsencrypt
      guest: /letsencrypt

nginx_haproxy_ip_template.yml（自定义模板，解析用户ip地址）

run:
  - exec:
     cmd:
       #- sed -i "/sendfile on;/a \ \ set_real_ip_from 50.100.200.100;\n\ \ real_ip_header X-Forwarded-For;" /etc/nginx/conf.d/discourse.conf
       - sed -i "/sendfile on;/a \ \ set_real_ip_from 50.100.200.100;\n\ \ real_ip_header proxy_protocol;" /etc/nginx/conf.d/discourse.conf
       - sed -i 's/http2;/http2 proxy_protocol;/' /etc/nginx/conf.d/discourse.conf

web.letsencrypt.ssl.template.yml（修改支持顶级域名（zeppos.tech）ssl证书自动申请）

        issue_cert() {
          LE_WORKING_DIR="${LETSENCRYPT_DIR}" $$ENV_LETSENCRYPT_DIR/acme.sh --issue $2 -d $$ENV_DISCOURSE_HOSTNAME -d zeppos.tech --keylength $1 -w /var/www/discourse/public
        }

        LE_WORKING_DIR="${LETSENCRYPT_DIR}" $$ENV_LETSENCRYPT_DIR/acme.sh \
          --installcert \
          -d $$ENV_DISCOURSE_HOSTNAME -d zeppos.tech \
          --fullchainpath /shared/ssl/$$ENV_DISCOURSE_HOSTNAME.cer \
          --keypath /shared/ssl/$$ENV_DISCOURSE_HOSTNAME.key \
          --reloadcmd "sv reload nginx"

        LE_WORKING_DIR="${LETSENCRYPT_DIR}" $$ENV_LETSENCRYPT_DIR/acme.sh \
          --installcert --ecc \
          -d $$ENV_DISCOURSE_HOSTNAME -d zeppos.tech \
          --fullchainpath /shared/ssl/$$ENV_DISCOURSE_HOSTNAME_ecc.cer \
          --keypath /shared/ssl/$$ENV_DISCOURSE_HOSTNAME_ecc.key \
          --reloadcmd "sv reload nginx"

构建论坛

构建指令位于工程根目录执行

配置containers文件夹所有用户使用权限，用于将配置文件通过MobaXterm复制到文件夹中

sudo chmod 777 containers

构建data

sudo ./launcher rebuild data

构建web_only

sudo ./launcher rebuild web_only

公网服务器部署tcp负载均衡

使用tcp负载均衡代替之前的iptables转发，通过代理协议send-proxy-v2，配置后端nginx解析公网用户真实ip地址。

# 安装haproxy
sudo apt update
sudo apt install haproxy

# 配置
sudo vi /etc/haproxy/haproxy.cfg

# 检查配置语法
haproxy -c -f /etc/haproxy/haproxy.cfg

# 重启应用
sudo systemctl restart haproxy

# log系统
sudo vi /etc/rsyslog.conf
sudo touch /var/log/haproxy.log
sudo chmod a+w /var/log/haproxy.log
sudo systemctl restart rsyslog

系统log配置

local0.*    /var/log/haproxy.log

服务端haproxy配置

global
    log 127.0.0.1 local0

frontend front_443
    bind *:443
    mode tcp
    option tcplog
    timeout client 60s
    use_backend backend_443
    log global

backend backend_443
    mode tcp
    timeout connect 60s
    timeout server 120s
    server server_443 50.100.200.101:443 send-proxy-v2

frontend front_80
    bind *:80
    mode tcp
    option tcplog
    timeout client 60s
    use_backend backend_80
    log global

backend backend_80
    mode tcp
    timeout connect 60s
    timeout server 120s
    server server_80 50.100.200.101:80

frontend front_25
    bind *:25
    mode tcp
    option tcplog
    timeout client 60s
    use_backend backend_25
    log global

backend backend_25
    mode tcp
    timeout connect 60s
    timeout server 120s
    server server_25 50.100.200.101:25

CSP安全策略

cdn服务器不能使用Rocket Loader™优化
源服务器关闭CSP策略

image
image748×131 2.17 KB

解析用户真实ip地址

防止之前iptables转发导致相同ip地址高并发触发论坛系统防攻击拦截

image
image906×403 14 KB

image
image551×808 74.1 KB

cdn全球加速

image
image1269×884 77.9 KB

image
image1239×807 85.6 KB

支持顶级域名ssl访问

非ssl访问会自动转换为ssl访问，顶级域名ssl访问会转换为子域名访问，之前不支持，因google、openai要求网站的地址必须是顶级域名，做升级支持

正常子域名ssl访问
https://www.zeppos.tech

顶级域名ssl访问
https://zeppos.tech