rsync性能测试+多节点配置

一、参数性能测试

image.png
image.png
image.png
image.png
image.png
image.png
image.png

测试结果 rsync -r -c --progress /rsync/2dir/ 192.168.133.123:/rsync/1dir/

加上需求的参数（不影响性能）： rsync -cgroupDt

二、各节点配置同步目录（支持多台）

各节点配置情况（相同操作）

yum -y install rsync 
yum -y install  inotify-tools 


脚本：
#!/bin/bash 


src=/rsync/
#src为inotifywait监控目录/同步目录
/usr/bin/inotifywait -mrq --timefmt '%d/%m/%y %H:%M' --format '%T %w%f%e' -e create,modify,move $src |  while read file
do
      txt=`echo $file | awk '{print $3}'`
      filename=`echo $txt | cut -c1-10` 
      filetype=`echo $txt | cut -c 8 `
      if [ $filetype != '.' ];then
          rsync -cgroupDt      $filename*   192.168.133.123:/rsync/ &>/dev/null  &
          rsync -cgroupDt      $filename*   192.168.133.124:/rsync/ &>/dev/null  &
#以上主机为另外两台同步服务器节点（几台写几个）
      fi
      echo "  ${file} was rsynced" >>/opt/soft/log/rsync.log 2>&1
#同步日志存放在/opt/soft/log/rsync.log
done

运行以上脚本 加 & 放在后台执行即可实现实时同步

三、nginx图床配置

代理端配置nginx.conf

http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 2048;

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;

    # Load modular configuration files from the /etc/nginx/conf.d directory.
    # See http://nginx.org/en/docs/ngx_core_module.html#include
    # for more information.
    include /etc/nginx/conf.d/*.conf;

    server {
           listen 80;
           server_name localhost;
           proxy_set_header X-Forwarded-Host $host;
           proxy_set_header X-Forwarded-Server $host;
           proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
           location / {
               #这里的“bal”应与已存在的某一策略名称一致
               proxy_pass http://bal;
           }
       }
       #配置均衡策略，"bal"为策略名称
       upstream bal {
           #轮询
           server  192.168.133.122;
           server  192.168.133.123;
           server  192.168.133.124;
       }
}

nginx节点配置nginx.conf

    server {
        listen       80 default_server;
        listen       [::]:80 default_server;
        server_name  _;
        root         /rsync;   #nginx工作目录
        #只匹配图片
        location ~ .*\.(gif|jpg|jpeg|png|jfif)$ {
        autoindex    on;

}
        # Load configuration files for the default server block.
        include /etc/nginx/default.d/*.conf;

        location / {
        }

        error_page 404 /404.html;
            location = /40x.html {
        }

        error_page 500 502 503 504 /50x.html;
            location = /50x.html {
        }
    }

反爬虫配置

#禁止Scrapy等工具的抓取
if ($http_user_agent ~* (Scrapy|Curl|HttpClient)) {
     return 403;
}
 
#禁止指定UA及UA为空的访问
if ($http_user_agent ~ "WinHttp|WebZIP|FetchURL|node-superagent|java/|
FeedDemon|Jullo|JikeSpider|Indy Library|Alexa Toolbar|AskTbFXTV|AhrefsBot|
CrawlDaddy|Java|Feedly|Apache-HttpAsyncClient|UniversalFeedParser|ApacheBench|
Microsoft URL Control|Swiftbot|ZmEu|oBot|jaunty|Python-urllib|
lightDeckReports Bot|YYSpider|DigExt|HttpClient|MJ12bot|heritrix|EasouSpider|Ezooms|BOT/0.1|
YandexBot|FlightDeckReports|Linguee Bot|^$" ) {
     return 403;             
}