keepalived实现nginx负载均衡机高可用目录

  • keepalived实现nginx负载均衡机高可用
    • 配置web界面
    • 配置nginx负载均衡
    • 配置keepalived高可用
    • 编写脚本
    • 配置keepalived加入监控脚本的配置
    • 测试

环境说明:

系统主机名IP服务
centos8master192.168.111.141nginx
keepalived
centos8backup192.168.111.142nginx
keepalived
centos8apache192.168.111.143httpd
centos8nginx192.168.111.144nginx

配置web界面

apache

//修改名字[root@localhost ~]# hostnamectl set-hostname apache[root@localhost ~]# bash[root@apache ~]# //关闭防火墙和selinux[root@apache ~]# setenforce 0[root@apache ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config[root@apache ~]# systemctl disable --now firewalld[root@apache ~]# reboot//配置yum源[root@apache ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo[root@apache ~]# sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo//安装apache,主页内容为apache[root@apache ~]# yum -y install httpd[root@apache ~]# echo "apache" > /var/www/html/index.html[root@apache ~]# systemctl enable --now httpd[root@apache ~]# ss -anltState        Recv-Q        Send-Q               Local Address:Port               Peer Address:Port       Process       LISTEN       0             128                        0.0.0.0:22                      0.0.0.0:*                        LISTEN       0             128                              *:80                            *:*                        LISTEN       0             128                           [::]:22                         [::]:*                        [root@apache ~]# curl 192.168.111.143apache

nginx

//修改名字[root@localhost ~]# hostnamectl set-hostname nginx[root@localhost ~]# bash[root@nginx ~]# //关闭防火墙和selinux[root@nginx ~]# setenforce 0[root@nginx ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config[root@nginx ~]# systemctl disable --now firewalld[root@nginx ~]# reboot//配置yum源[root@nginx ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo[root@nginx ~]# sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo//安装nginx,主页内容为nginx[root@nginx ~]# yum -y install nginx[root@nginx ~]# echo "nginx" > /usr/share/nginx/html/index.html[root@nginx ~]# systemctl enable --now nginx[root@nginx ~]# ss -anltState        Recv-Q        Send-Q               Local Address:Port               Peer Address:Port       Process       LISTEN       0             128                        0.0.0.0:80                      0.0.0.0:*                        LISTEN       0             128                        0.0.0.0:22                      0.0.0.0:*                        LISTEN       0             128                           [::]:80                         [::]:*                        LISTEN       0             128                           [::]:22                         [::]:*                        [root@nginx ~]# curl 192.168.111.144nginx

配置nginx负载均衡

master

//修改名字[root@localhost ~]# hostnamectl set-hostname master[root@localhost ~]# bash[root@master ~]# //关闭防火墙和selinux[root@master ~]# setenforce 0[root@master ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config[root@master ~]# systemctl disable --now firewalld[root@master ~]# reboot//配置yum源[root@master ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo[root@master ~]# sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo//创建用户[root@master ~]# useradd -rMs /sbin/nologin nginx//安装依赖包[root@master ~]# dnf -y install pcre-devel openssl openssl-devel gd-devel gcc gcc-c++ make wget vim//编译安装nginx[root@backup ~]# wget http://nginx.org/download/nginx-1.22.0.tar.gz[root@master ~]# tar xf nginx-1.22.0.tar.gz [root@master ~]# cd nginx-1.22.0[root@master nginx-1.22.0]# ./configure \ --prefix=/usr/local/nginx \ --user=nginx \ --group=nginx \ --with-debug \ --with-http_ssl_module \ --with-http_realip_module \ --with-http_gunzip_module \ --with-http_gzip_static_module \ --with-http_stub_status_module [root@master nginx-1.22.0]# make -j $(grep 'processor' /proc/cpuinfo | wc -l) && make install//配置环境变量[root@master ~]# echo "export PATH=$PATH:/usr/local/nginx/sbin" > /etc/profile.d/nginx.sh[root@master ~]# source /etc/profile.d/nginx.sh                //配置system启动服务[root@master ~]# vim /usr/lib/systemd/system/nginx.service[Unit]Description=nginx server daemonAfter=network.target  [Service]Type=forkingExecStart=/usr/local/nginx/sbin/nginxExecStop=/usr/local/nginx/sbin/nginx -s stopExecReload=/bin/kill -HUP \$MAINPID [Install]WantedBy=multi-user.target[root@nginx ~]# systemctl daemon-reload//启动nginx[root@master ~]# systemctl enable --now nginx[root@master ~]# ss -anltState        Recv-Q       Send-Q               Local Address:Port               Peer Address:Port       Process       LISTEN       0            128                        0.0.0.0:80                      0.0.0.0:*                        LISTEN       0            128                        0.0.0.0:22                      0.0.0.0:*                        LISTEN       0            128                           [::]:22                         [::]:*               //修改配置文件,配置负载均衡[root@master ~]# vim /usr/local/nginx/conf/nginx.conf    upstream web {//http字段内    server 192.168.111.143;    server 192.168.111.144;    }    .........        location / {//修改配置文件        root html;        proxy_pass   http://web;        }[root@master ~]# systemctl restart nginx//查看负载均衡效果[root@master ~]# curl 192.168.111.141apache[root@master ~]# curl 192.168.111.141nginx[root@master ~]# curl 192.168.111.141apache[root@master ~]# curl 192.168.111.141nginx

backup

//修改名字[root@localhost ~]# hostnamectl set-hostname backup[root@localhost ~]# bash[root@backup ~]# //关闭防火墙和selinux[root@backup ~]# setenforce 0[root@backup ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config[root@backup ~]# systemctl disable --now firewalld[root@backup ~]# reboot//配置yum源[root@backup ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo[root@backup ~]# sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo//创建用户[root@backup ~]# useradd -rMs /sbin/nologin nginx//安装依赖包[root@backup ~]# dnf -y install pcre-devel openssl openssl-devel gd-devel gcc gcc-c++ make wget vim//编译安装nginx[root@backup ~]# wget http://nginx.org/download/nginx-1.22.0.tar.gz[root@backup ~]# tar xf nginx-1.22.0.tar.gz [root@backup ~]# cd nginx-1.22.0[root@backup nginx-1.22.0]# ./configure \ --prefix=/usr/local/nginx \ --user=nginx \ --group=nginx \ --with-debug \ --with-http_ssl_module \ --with-http_realip_module \ --with-http_gunzip_module \ --with-http_gzip_static_module \ --with-http_stub_status_module[root@backup nginx-1.22.0]# make -j $(grep 'processor' /proc/cpuinfo | wc -l) && make install//配置环境变量[root@backup ~]# echo "export PATH=$PATH:/usr/local/nginx/sbin" > /etc/profile.d/nginx.sh[root@backup ~]# source /etc/profile.d/nginx.sh                //配置system启动服务[root@backup ~]# vim /usr/lib/systemd/system/nginx.service[Unit]Description=nginx server daemonAfter=network.target  [Service]Type=forkingExecStart=/usr/local/nginx/sbin/nginxExecStop=/usr/local/nginx/sbin/nginx -s stopExecReload=/bin/kill -HUP \$MAINPID [Install]WantedBy=multi-user.target[root@backup ~]# systemctl daemon-reload//启动nginx[root@backup ~]# systemctl enable --now nginx[root@backup ~]# ss -anltState        Recv-Q        Send-Q               Local Address:Port               Peer Address:Port       Process       LISTEN       0             128                        0.0.0.0:80                      0.0.0.0:*                        LISTEN       0             128                        0.0.0.0:22                      0.0.0.0:*                        LISTEN       0             128                           [::]:22                         [::]:*                        //修改配置文件,配置负载均衡[root@backup ~]# vim /usr/local/nginx/conf/nginx.conf    upstream web {//http字段内    server 192.168.111.143;    server 192.168.111.144;    }    .........        location / {//修改配置文件        root html;        proxy_pass   http://web;        }[root@backup ~]# systemctl restart nginx//查看负载均衡效果[root@backup ~]# curl 192.168.111.142apache[root@backup ~]# curl 192.168.111.142nginx[root@backup ~]# curl 192.168.111.142apache[root@backup ~]# curl 192.168.111.142nginx

配置keepalived高可用

master

//首先安装keepalived[root@master ~]# yum -y install keepalived //编辑配置文件,并启动服务[root@master ~]# mv /etc/keepalived/keepalived.conf{,.bak}[root@master ~]# vim /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs {   router_id lb01} vrrp_instance VI_1 {    state MASTER    interface ens33    virtual_router_id 51    priority 100    advert_int 1    authentication {        auth_type PASS        auth_pass 123456    }    virtual_ipaddress {        192.168.111.250    }} virtual_server 192.168.111.250 80 {    delay_loop 6    lb_algo rr    lb_kind DR    persistence_timeout 50    protocol TCP     real_server 192.168.111.141 80 {        weight 1        TCP_CHECK {            connect_port 80            connect_timeout 3            nb_get_retry 3            delay_before_retry 3        }    }     real_server 192.168.111.142 80 {        weight 1        TCP_CHECK {            connect_port 80            connect_timeout 3            nb_get_retry 3            delay_before_retry 3        }    }}[root@master ~]# systemctl enable --now keepalived//通过虚拟IP访问[root@master ~]# ip a1: lo:  mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00    inet 127.0.0.1/8 scope host lo       valid_lft forever preferred_lft forever    inet6 ::1/128 scope host        valid_lft forever preferred_lft forever2: ens33:  mtu 1500 qdisc fq_codel state UP group default qlen 1000    link/ether 00:0c:29:50:34:72 brd ff:ff:ff:ff:ff:ff    inet 192.168.111.141/24 brd 192.168.111.255 scope global dynamic noprefixroute ens33       valid_lft 1500sec preferred_lft 1500sec    inet 192.168.111.250/32 scope global ens33       valid_lft forever preferred_lft forever    inet6 fe80::20c:29ff:fe50:3472/64 scope link noprefixroute        valid_lft forever preferred_lft forever //通过虚拟ip访问//一定要把backup端的nginx服务关掉才能访问,[root@master ~]# curl 192.168.111.250apache[root@master ~]# curl 192.168.111.250nginx[root@master ~]# curl 192.168.111.250apache[root@master ~]# curl 192.168.111.250nginx

backup

//首先安装keepalived[root@backup ~]# dnf -y install keepalived //编辑配置文件,并启动服务[root@backup ~]# mv /etc/keepalived/keepalived.conf{,.back}[root@backup ~]# vim /etc/keepalived/keepalived.conf! Configuration File for keepalived global_defs {   router_id lb02} vrrp_instance VI_1 {    state BACKUP    interface ens33    virtual_router_id 50    priority 90    advert_int 1    authentication {        auth_type PASS        auth_pass 123456    }    virtual_ipaddress {        192.168.111.250    }} virtual_server 192.168.111.250 80 {    delay_loop 6    lb_algo rr    lb_kind DR    persistence_timeout 50    protocol TCP     real_server 192.168.111.141 80 {        weight 1        TCP_CHECK {            connect_port 80            connect_timeout 3            nb_get_retry 3            delay_before_retry 3        }    }     real_server 192.168.111.142 80 {        weight 1        TCP_CHECK {            connect_port 80            connect_timeout 3            nb_get_retry 3            delay_before_retry 3        }    }}[root@backup ~]# systemctl enable --now keepalived

编写脚本

master

[root@master ~]# mkdir /scripts[root@master ~]# cd /scripts/[root@master scripts]# vim check_nginx.sh#!/bin/bashnginx_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bnginx\b'|wc -l)if [ $nginx_status -lt 1 ];then    systemctl stop keepalivedfi[root@master scripts]# vim notify.sh#!/bin/bashVIP=$2case "$1" in  master)        nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bnginx\b'|wc -l)        if [ $nginx_status -lt 1 ];then            systemctl start nginx        fi  ;;  backup)        nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bnginx\b'|wc -l)        if [ $nginx_status -gt 0 ];then            systemctl stop nginx        fi  ;;  *)        echo "Usage:$0 master|backup VIP"  ;;esac[root@master scripts]# chmod +x check_haproxy.sh notify.sh 

backup

[root@backup ~]# mkdir /scripts[root@backup ~]# cd /scripts/[root@backup scripts]# scp root@192.168.111.141:/scripts/notify.sh .

配置keepalived加入监控脚本的配置

master

[root@master ~]# vim /etc/keepalived/keepalived.conf! Configuration File for keepalived global_defs {   router_id lb01} vrrp_script nginx_check {//添加    script "/scripts/check_nginx.sh"    interval 1    weight -20}vrrp_instance VI_1 {    state MASTER    interface ens33    virtual_router_id 51    priority 100    advert_int 1    authentication {        auth_type PASS        auth_pass 123456    }    virtual_ipaddress {        192.168.111.250    }    track_script {//添加        haproxy_check    }    notify_master "/scripts/notify.sh master 192.168.111.250"} virtual_server 192.168.111.250 80 {    delay_loop 6    lb_algo rr    lb_kind DR    persistence_timeout 50    protocol TCP     real_server 192.168.111.141 80 {        weight 1        TCP_CHECK {            connect_port 80            connect_timeout 3            nb_get_retry 3            delay_before_retry 3        }    }     real_server 192.168.111.142 80 {        weight 1        TCP_CHECK {            connect_port 80            connect_timeout 3            nb_get_retry 3            delay_before_retry 3        }    }}[root@master scripts]# systemctl restart keepalived

backup

[root@backup ~]# vim /etc/keepalived/keepalived.conf! Configuration File for keepalived global_defs {   router_id lb02} vrrp_instance VI_1 {    state BACKUP    interface ens33    virtual_router_id 50    priority 90    advert_int 1    authentication {        auth_type PASS        auth_pass 123456    }    virtual_ipaddress {        192.168.111.250    }    notify_master "/scripts/notify.sh master 192.168.111.250"//添加    notify_backup "/scripts/notify.sh backup 192.168.111.250"} virtual_server 192.168.111.250 80 {    delay_loop 6    lb_algo rr    lb_kind DR    persistence_timeout 50    protocol TCP     real_server 192.168.111.141 80 {        weight 1        TCP_CHECK {            connect_port 80            connect_timeout 3            nb_get_retry 3            delay_before_retry 3        }    }     real_server 192.168.111.142 80 {        weight 1        TCP_CHECK {            connect_port 80            connect_timeout 3            nb_get_retry 3            delay_before_retry 3        }    }}[root@backup ~]# systemctl restart keepalived

测试

模拟nginx服务故障

//master端[root@master ~]# curl 192.168.111.250apache[root@master ~]# curl 192.168.111.250nginx[root@master ~]# ip a1: lo:  mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00    inet 127.0.0.1/8 scope host lo       valid_lft forever preferred_lft forever    inet6 ::1/128 scope host        valid_lft forever preferred_lft forever2: ens33:  mtu 1500 qdisc fq_codel state UP group default qlen 1000    link/ether 00:0c:29:50:34:72 brd ff:ff:ff:ff:ff:ff    inet 192.168.111.141/24 brd 192.168.111.255 scope global dynamic noprefixroute ens33       valid_lft 1601sec preferred_lft 1601sec    inet 192.168.111.250/32 scope global ens33       valid_lft forever preferred_lft forever    inet6 fe80::20c:29ff:fe50:3472/64 scope link noprefixroute        valid_lft forever preferred_lft forever[root@master ~]# systemctl stop nginx[root@master ~]# ip a1: lo:  mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00    inet 127.0.0.1/8 scope host lo       valid_lft forever preferred_lft forever    inet6 ::1/128 scope host        valid_lft forever preferred_lft forever2: ens33:  mtu 1500 qdisc fq_codel state UP group default qlen 1000    link/ether 00:0c:29:50:34:72 brd ff:ff:ff:ff:ff:ff    inet 192.168.111.141/24 brd 192.168.111.255 scope global dynamic noprefixroute ens33       valid_lft 1591sec preferred_lft 1591sec    inet6 fe80::20c:29ff:fe50:3472/64 scope link noprefixroute        valid_lft forever preferred_lft forever//backup端[root@backup ~]# systemctl start nginx    //前面把服务关了这里启动一下[root@backup ~]# ip a1: lo:  mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00    inet 127.0.0.1/8 scope host lo       valid_lft forever preferred_lft forever    inet6 ::1/128 scope host        valid_lft forever preferred_lft forever2: ens33:  mtu 1500 qdisc fq_codel state UP group default qlen 1000    link/ether 00:0c:29:07:42:65 brd ff:ff:ff:ff:ff:ff    inet 192.168.111.142/24 brd 192.168.111.255 scope global dynamic noprefixroute ens33       valid_lft 947sec preferred_lft 947sec    inet 192.168.111.250/32 scope global ens33       valid_lft forever preferred_lft forever    inet6 fe80::20c:29ff:fe07:4265/64 scope link noprefixroute        valid_lft forever preferred_lft forever[root@backup ~]# curl 192.168.111.250apache[root@backup ~]# curl 192.168.111.250nginx[root@backup ~]# curl 192.168.111.250apache[root@backup ~]# curl 192.168.111.250nginx

启动master端的nginx服务

//master端[root@master scripts]# systemctl start nginx[root@master scripts]# systemctl restart keepalived[root@master ~]# ip a1: lo:  mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00    inet 127.0.0.1/8 scope host lo       valid_lft forever preferred_lft forever    inet6 ::1/128 scope host        valid_lft forever preferred_lft forever2: ens33:  mtu 1500 qdisc fq_codel state UP group default qlen 1000    link/ether 00:0c:29:50:34:72 brd ff:ff:ff:ff:ff:ff    inet 192.168.111.141/24 brd 192.168.111.255 scope global dynamic noprefixroute ens33       valid_lft 1223sec preferred_lft 1223sec    inet 192.168.111.250/32 scope global ens33       valid_lft forever preferred_lft forever    inet6 fe80::20c:29ff:fe50:3472/64 scope link noprefixroute        valid_lft forever preferred_lft forever//backup端[root@backup ~]# ip a1: lo:  mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00    inet 127.0.0.1/8 scope host lo       valid_lft forever preferred_lft forever    inet6 ::1/128 scope host        valid_lft forever preferred_lft forever2: ens33:  mtu 1500 qdisc fq_codel state UP group default qlen 1000    link/ether 00:0c:29:07:42:65 brd ff:ff:ff:ff:ff:ff    inet 192.168.111.142/24 brd 192.168.111.255 scope global dynamic noprefixroute ens33       valid_lft 1632sec preferred_lft 1632sec    inet6 fe80::20c:29ff:fe07:4265/64 scope link noprefixroute        valid_lft forever preferred_lft forever