log.io+inotify+rsync 做日志监控

一、rsync+inotify部署

#log-prepublish为slave   elastic为master  elastic上的日志通过logstash从服务器抓取
[root@log-prepublish ~]# yum install rsync -y
[root@log-prepublish ~]# useradd rsync -s /sbin/nologin  -M #添加rsync用户
[root@log-prepublish ~]# grep rsync /etc/passwd
rsync:x:2004:2004::/home/rsync:/sbin/nologin
[root@log-prepublish ~]# mkdir /backup   #创建rsync daemon工作模式的模块目录
[root@log-prepublish ~]# chown rsync.rsync /backup/   #更改模块目录的用户组
[root@log-prepublish ~]# cat /etc/rsyncd.conf
uid = rsync
gid = rsync
use chroot = no
max connections = 200
timeout = 300
pid file = /var/run/rsyncd.pid
lock file = /var/run/rsync.lock
log file = /var/log/rsyncd.log
[backup]
path = /backup/
ignore errors
read only = false
list = false
hosts allow = 192.168.1.0/24
hosts deny = 0.0.0.0/32
auth users = rsync_backup
secrets file = /etc/rsync.password

[root@log-prepublish ~]# echo "rsync_backup:123456" >/etc/rsync.password
[root@log-prepublish ~]# chmod 600 /etc/rsync.password 
[root@log-prepublish ~]# rsync --daemon

#master服务器上

[root@elastic ~]# wget http://github.com/downloads/rvoicilas/inotify-tools/inotify-tools-3.14.tar.gz  #下载inotify源码包
[root@elastic ~]# tar zxf inotify-tools-3.14.tar.gz
[root@elastic ~]# cd inotify-tools-3.14
[root@elastic inotify-tools-3.14]#./configure --prefix=/usr/local/inotify-3.14 
[root@ elastic  inotify-tools-3.14]# make && make install

[root@elastic ~]# cd /usr/local/inotify-3.14/
[root@elastic inotify-3.14]# cat inotify.sh
#!/bin/bash
#para
host01=192.168.1.92  #inotify-slave的ip地址
src=/tzgData/logstash/prepublish/       #本地监控的目录
dst=backup         #inotify-slave的rsync服务的模块名
user=rsync_backup      #inotify-slave的rsync服务的虚拟用户
rsync_passfile=/etc/rsync.password   #本地调用rsync服务的密码文件
inotify_home=/usr/local/inotify-3.14    #inotify的安装目录
#judge
if [ ! -e "$src" ] \
|| [ ! -e "${rsync_passfile}" ] \
|| [ ! -e "${inotify_home}/bin/inotifywait" ] \
|| [ ! -e "/usr/bin/rsync" ];
then
echo "Check File and Folder"
exit 9
fi
${inotify_home}/bin/inotifywait -mrq --timefmt '%d/%m/%y %H:%M' --format '%T %w%f' -e close_write,delete,create,attrib $src \
| while read file
do
#  rsync -avzP --delete --timeout=100 --password-file=${rsync_passfile} $src $user@$host01::$dst >/dev/null 2>&1
cd $src && rsync -aruz -R --delete ./  --timeout=100 $user@$host01::$dst --password-file=${rsync_passfile} >/dev/null 2>&1
done
exit 0

[root@elastic inotify-3.14] sh inotify.sh &  #将脚本加入后台执行

二、部署log.io

[root@log-prepublish ~]# yum install gcc gcc-c++ openssl-devel  pkgconfig -y #安装依赖包
[root@log-prepublish ~]# yum install epel-release -y
[root@log-prepublish ~]# yum install nodejs -y
[root@log-prepublish ~]# yum install npm -y
[root@log-prepublish ~]# node -v
v6.9.4
[root@log-prepublish ~]# npm -v
3.10.10
[root@log-prepublish ~]# npm install -g log.io --user "root"  #安装log.io
[root@log-prepublish ~]# cat /root/.log.io/harvester.conf
exports.config = {
  nodeName: "prepublish",
  logStreams: {
    rest: [
      "/backup/rest.prepublishtomcat-catalina-out",
    ],
    web: [
      "/backup/pay.prepublishtomcat-catalina-out",
    ]
  },
  server: {
    host: '192.168.1.92',
    port: 28777
  }
}

[root@log-prepublish ~]# log.io-server start   #启动服务端
[root@log-prepublish ~]# log.io-harvester start  #启动客户端 （客户端服务器在同一台服务器上）


# 测试发现使用 nohup  & 也无法在后台运行不中断服务，所以使用screen开启服务


[root@log-prepublish prepublish]# screen -ls
There are screens on:
    2200.harvester  (Detached)
    2173.logioserver    (Detached)
2 Sockets in /var/run/screen/S-root.