一、需求背景
发现公司好多项目都用到memched。并且都是单点,memched服务器 挂了就会影响业务,于是百度之,发现memched不可以集群,也没有看到有什么高可用方案,但是发现了repcached(memched主从复制),结合LVS的NAT模式不同端口转发实现memched双机HA,本文章记录本人的部署过程,从属实战,没有文字性的说明,欢迎各位指点。
二、环境:
1、系统环境:
-
CentOS release 6.4 (Final)
2、网络环境
-
调度机(master):
-
vip:172.28.26.100
-
vip1:172.28.16.100(lvs转发网关 )
-
eth1:172.28.26.101 (内网)
-
eth2:172.28.16.101(lvs转发网段)
-
调度机(backup):
-
eth1:172.28.26.99 (内网)
-
eth2:172.28.16.99(lvs转发网段)
-
memched备节点:
-
eth1:172.28.26.102 (内网)
-
eth2:172.28.16.102(lvs转发网段)
-
memched主节点:
-
eth1:172.28.26.103 (内网)
-
eth2:172.28.16.103(lvs转发网段)
3、路由策略
-
调度机(master):
-
echo '201 eth1' >> /etc/iproute2/rt_tables
-
echo 'default table eth1 via 172.28.26.1 dev eth1' > /etc/sysconfig/network-scripts/route-eth1
-
echo 'from 172.28.26.101/255.255.255.255 table eth1' > /etc/sysconfig/network-scripts/rule-eth1
-
echo 'from 172.28.26.100/255.255.255.255 table eth1' >> /etc/sysconfig/network-scripts/rule-eth1
-
echo 'from 172.28.16.101/32 table ZW_LVS_LAN' > /etc/sysconfig/network-scripts/rule-eth2
-
memched节点1:
-
echo 'default table ZW_LVS_LAN via 172.28.16.100' > /etc/sysconfig/network-scripts/route-eth2
-
echo 'from 172.28.16.0/24 table ZW_LVS_LAN' > /etc/sysconfig/network-scripts/rule-eth2
-
echo '202 ZW_LVS_LAN' >> /etc/iproute2/rt_tables
-
memched节点2:
-
echo 'default table ZW_LVS_LAN via 172.28.16.100' > /etc/sysconfig/network-scripts/route-eth2
-
echo 'from 172.28.16.0/24 table ZW_LVS_LAN' > /etc/sysconfig/network-scripts/rule-eth2
-
echo '202 ZW_LVS_LAN' >> /etc/iproute2/rt_tables
4、内核参数
-
fs.file-max = 1000000
-
kernel.core_uses_pid = 1
-
kernel.msgmax = 1048560
-
kernel.msgmnb = 1073741824
-
kernel.shmall = 4294967296
-
kernel.shmmax = 68719476736
-
kernel.sysrq = 0
-
net.core.netdev_max_backlog = 1048576
-
net.core.rmem_default = 2097152
-
net.core.rmem_max = 16777216
-
net.core.somaxconn = 1048576
-
net.core.wmem_default = 2097152
-
net.core.wmem_max = 16777216
-
net.ipv4.conf.default.accept_source_route = 0
-
net.ipv4.conf.default.rp_filter = 1
-
net.ipv4.ip_forward = 1
-
net.ipv4.ip_local_port_range = 1024 65000
-
net.ipv4.neigh.default.gc_thresh1 = 10240
-
net.ipv4.neigh.default.gc_thresh2 = 40960
-
net.ipv4.neigh.default.gc_thresh3 = 81920
-
net.ipv4.tcp_fin_timeout = 1
-
net.ipv4.tcp_keepalive_intvl = 15
-
net.ipv4.tcp_keepalive_probes = 5
-
net.ipv4.tcp_keepalive_time = 30
-
net.ipv4.tcp_max_orphans = 3276800
-
net.ipv4.tcp_max_syn_backlog = 1048576
-
net.ipv4.tcp_max_tw_buckets = 50000
-
net.ipv4.tcp_mem = 94500000 915000000 927000000
-
net.ipv4.tcp_orphan_retries = 3
-
net.ipv4.tcp_reordering = 5
-
net.ipv4.tcp_retrans_collapse = 0
-
net.ipv4.tcp_retries2 = 5
-
net.ipv4.tcp_rmem = 4096 87380 4194304
-
net.ipv4.tcp_sack = 1
-
net.ipv4.tcp_synack_retries = 1
-
net.ipv4.tcp_syncookies = 0
-
net.ipv4.tcp_syn_retries = 1
-
net.ipv4.tcp_timestamps = 1
-
net.ipv4.tcp_tw_recycle = 1
-
net.ipv4.tcp_tw_reuse = 1
-
net.ipv4.tcp_window_scaling = 1
-
net.ipv4.tcp_wmem = 4096 16384 4194304
-
net.ipv6.conf.all.disable_ipv6 = 1
-
net.ipv6.conf.default.disable_ipv6 = 1
-
net.ipv4.conf.eth0.rp_filter = 1 #memched节点不需要
-
net.ipv4.conf.eth1.rp_filter = 1 #memched节点不需要
-
net.ipv4.conf.eth2.rp_filter = 1 #memched节点不需要
-
net.ipv4.conf.all.rp_filter = 0 #memched节点不需要
三、服务安装
1、调度机(master、backup)
-
yum -y install ipvsadm keepalived sendmail
2、memched节点1、memched节点2
-
wget http://memcached./files/memcached-1.4.15.tar.gz
-
wget http://www./~provos/libevent-1.4.13-stable.tar.gz
-
wget http://downloads./repcached/memcached-1.2.8-repcached-2.2.tar.gz
-
tar -zxvf libevent-1.4.13-stable.tar.gz
-
cd libevent-1.4.13-stable
-
./configure --prefix=/usr/local
-
make
-
make install
-
-
tar -zxvf memcached-1.4.15.tar.gz
-
cd memcached-1.4.15
-
./configure --with-libevent=/usr/local
-
make
-
make install
-
-
tar -zxvf memcached-1.2.8-repcached-2.2.tar.gz
-
cd memcached-1.2.8-repcached-2.2
-
./configure --enable-replication
-
make
-
make install
四、keepalived配置(master、backup)
[root@LVS1 keepalived]# cat /etc/keepalived/gobal_module
-
! global configure file
-
-
global_defs {
-
notification_email {
-
navyaijm@qq.com
-
}
-
notification_email_from navyaijm@qq.com
-
smtp_server 127.0.0.1
-
smtp_connect_timeout 30
-
router_id ZH_DG_LVS1
-
}
-
-
[root@LVS1 keepalived]# cat /etc/keepalived/keepalived.conf
-
! Keepalived main configure file
-
include /etc/keepalived/gobal_module
-
include /etc/keepalived/vrrpd_module
-
include /etc/keepalived/lvs_module
-
[root@LVS1 keepalived]# cat /etc/keepalived/vrrpd_module
-
vrrp_sync_group ZH_DG_Memcached1 {
-
group {
-
ZH_DG_WAN1
-
ZH_DG_LAN1
-
}
-
smtp_alter
-
}
-
-
vrrp_instance ZH_DG_WAN1 {
-
state MASTER (从上为:backup)
-
interface eth1
-
virtual_router_id 10 (从为:9)
-
priority 100
-
advert_int 1
-
authentication {
-
auth_type PASS
-
auth_pass 08756CD0
-
}
-
virtual_ipaddress {
-
172.28.26.100 # vip
-
}
-
}
-
-
-
vrrp_instance ZH_DG_LAN2 {
-
state MASTER (从上为:backup)
-
interface eth2
-
virtual_router_id 10 (从为:9)
-
priority 100
-
advert_int 1
-
authentication {
-
auth_type PASS
-
auth_pass 6F8DBC2E
-
}
-
virtual_ipaddress {
-
172.28.16.100 # Memcached 虚拟网关
-
}
-
}
[root@LVS1 keepalived]# cat /etc/keepalived/lvs_module
-
virtual_server 172.28.26.100 11211 {
-
delay_loop 6
-
lb_algo wrr
-
lb_kind NAT
-
persistence_timeout 60
-
protocol TCP
-
-
include /etc/keepalived/realserver/172.28.26.102_11234.conf
-
include /etc/keepalived/realserver/172.28.26.103_11233.conf
-
}
-
-
-
[root@LVS1 keepalived]# cat /etc/keepalived/realserver/172.28.26.102_11234.conf
-
real_server 172.28.16.102 11234 {
-
weight 1 #权重
-
inhibit_on_failure #故障会修改权重为零
-
TCP_CHECK {
-
connect_timeout 10 #10秒无响应超时
-
nb_get_retry 3 #重连次数
-
delay_before_retry 3 #重连间隔,单位为秒
-
connect_port 11234 #检测端口
-
}
-
}
-
-
-
[root@LVS1 keepalived]# cat /etc/keepalived/realserver/172.28.26.103_11233.conf
-
real_server 172.28.16.103 11233 {
-
weight 1 #权重
-
inhibit_on_failure #故障会修改权重为零
-
TCP_CHECK {
-
connect_timeout 10 #10秒无响应超时
-
nb_get_retry 3 #重连次数
-
delay_before_retry 3 #重连间隔,单位为秒
-
connect_port 11233 #检测端口
-
}
-
}
五、启动服务
1、keepalived启动(master和backup)
-
/etc/init.d/keepalived start
memcached的启动
主节点:
-
/usr/local/bin/memcached -d -v -l 0.0.0.0 -p 11233 -X 11244 -uroot (11244是数据同步端口)
备节点:
-
/usr/local/bin/memcached -d -v -l 0.0.0.0 -p 11234 -uroot -x 172.28.16.103 -X 11244(11234是服务监听端口,11244是监听数据同步端口;-x 指定masterIP,-X指定master数据同步监听端口)
-
PS:
-
1、如果启动的时候报错如下:
-
[root@yw_memcached_slave ~]#/usr/local/bin/memcached -d -v -l 0.0.0.0 -p 11233 -X 11244 -uroot
-
/usr/local/bin/memcached: error while loading shared libraries: libevent-1.4.so.2: cannot open shared object file: No such file or directory
-
请yum安装libeven
-
[root@yw_memcached_slave ~]# yum -y install libeven
-
2、repcached是日本人开发的实现memcached复制功能,它是一个单 master单 slave的方案,但它的 master/slave都是可读写的,而且可以相互同步,如果 master坏掉, slave侦测到连接断了,它会自动 listen而成为 master;而如果 slave坏掉, master也会侦测到连接断,它就会重新 listen等待新的 slave加入 ;master没有抢占功能,如果master挂掉再起来只能是从了,并且永远只能是从。
六、测试
1、写一个php测试文件
vi session.php
-
<?php
-
ini_set('session.save_handler', 'memcached');
-
ini_set('session.save_path', "172.28.26.100:11211");
-
session_start();
-
$_SESSION['aa'] = "bb";
-
-
echo session_id();
-
?>
2、用php执行这个文件会得到一个字符串,telnet 172.28.26.100 11211 或者telnet 172.28.26.103 11233再或者telnet 172.28.26.103 11234,如果能得到值说明session已经写入memched,down到任何一台memched数据不会丢。
本文出自 “为了梦想奋斗” 博客,转载请与作者联系!
原文地址:http://navyaijm.blog.51cto.com/4647068/1177508
|