xxl-job分布式任务调度平台

1、安装Jdk1.8+、Maven3.0+、MySQL5.7+

1.1、安装Jdk1.8+

yum -y install java-1.8.0-openjdk*

1.2、安装Maven3.0+

cd /usr/local/src
wget https://mirrors.cnnic.cn/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
tar xf apache-maven-3.6.3-bin.tar.gz -C /usr/local/
echo "export PATH=$PATH:/usr/local/apache-maven-3.6.3/bin" >>/etc/profile
source /etc/profile

1.3、安装MySQL5.7+

1.3.1、在/tmp目录下编写my.cnf文件

[client]
port    = 63307
socket  = /usr/local/mysql/mysql.sock
#character_set_client = utf8
#character_set_connection = utf8
#character_set_results = utf8
[mysql]
prompt="\u@host \R:\m:\s [\d]> "
no-auto-rehash

[mysqld]
user    = db-user
port    = 63307
basedir = /usr/local/mysql
datadir = /data/mysql/
socket  = /usr/local/mysql/mysql.sock
pid-file = /usr/local/mysql/mysql.pid
federated

character_set_server=utf8mb4
collation_server=utf8mb4_general_ci  #mysql服务排序规则
lower_case_table_names=1  #不区分大小写

skip_name_resolve = 1
open_files_limit    = 65535
sql_mode='' #配置sql模式
back_log = 1024 #MySql的连接数达到max_connections时，新来的请求将会被存在堆栈中，以等待某一连接释放资源，该堆栈的数量即back_log,可根据配置适当调高
max_connections = 2000
max_connect_errors = 100000

table_open_cache = 2048 #可根据配置适当调大
table_definition_cache = 2048
table_open_cache_instances = 16 #和CPU核心数有关 建议配置为CPU线程数量如果并发是1000则设置为32左右

thread_stack = 512K #mysql连接池 针对16G/32G的机器，一般设置 512K
max_allowed_packet = 128M

#network 释放sleep超时占用内存
interactive_timeout = 600
wait_timeout = 600

#performance for groupby sort
tmp_table_size = 64M
max_heap_table_size = 64M
innodb_doublewrite = on #innodb 二次写
innodb_adaptive_hash_index = on #自适应hash索引

#logs
log-error = /data/mysql/error.log
log_timestamps=system
#server configure for binlog
server-id = 101 #此处需修改
log-bin =mysql-bin
sync_binlog = 1
binlog_cache_size = 4096
max_binlog_cache_size = 2G
max_binlog_size = 1G
expire_logs_days = 10
master_info_repository = TABLE
relay_log_info_repository = TABLE
gtid_mode = on
enforce_gtid_consistency = 1
log_slave_updates
slave-rows-search-algorithms = 'INDEX_SCAN,HASH_SCAN'
binlog_format = row
binlog_checksum = 1
relay_log_recovery = 1
relay-log-purge = 1

innodb_thread_concurrency = 0
innodb_sync_spin_loops = 100
innodb_spin_wait_delay = 30
innodb_sync_array_size=10 #默认值为1 高并发建议提高
#transaction level
transaction_isolation = READ-COMMITTED

#innodb set
innodb_file_per_table = 1 #独立innodb空间
innodb_buffer_pool_size = 2G #最重要的内存设置参数建议为服务器70%
innodb_buffer_pool_instances = 1
innodb_buffer_pool_load_at_startup = 1  #是否保存innodb 缓存
innodb_buffer_pool_dump_at_shutdown = 1  #是否启动加载缓存
metadata_locks_hash_instances=16 #阻塞问题解决重要参数
#innodb_data_file_path = ibdata1:1G:autoextend
innodb_flush_log_at_trx_commit = 1
innodb_log_buffer_size = 32M
innodb_log_file_size = 2G
innodb_strict_mode = 0
innodb_log_files_in_group = 2
innodb_max_undo_log_size = 1G
innodb_support_xa = 1

# 根据您的服务器IOPS能力适当调整
# 一般配普通SSD盘的话，可以调整到 10000 - 20000
# 配置高端PCIe SSD卡的话，则可以调整的更高，比如 50000 - 80000
#innodb IO perfom set
innodb_io_capacity = 8000
innodb_io_capacity_max = 10000
innodb_flush_sync = 0 #如果服务器高可用可以设置为0 如果单点建议设置为1
innodb_flush_neighbors = 0
innodb_write_io_threads = 8 #根据CPU相对提高
innodb_read_io_threads = 8  #根据CPU相对提高
innodb_purge_threads = 4
innodb_page_cleaners = 4
innodb_open_files = 65535
innodb_max_dirty_pages_pct = 50
innodb_flush_method = O_DIRECT
innodb_lru_scan_depth = 4000
innodb_checksum_algorithm = crc32
innodb_lock_wait_timeout = 50
innodb_rollback_on_timeout = 1
innodb_print_all_deadlocks = 1
innodb_online_alter_log_max_size = 4G
internal_tmp_disk_storage_engine = InnoDB
innodb_stats_on_metadata = 0

# some var for MySQL 5.7 innodb
innodb_checksums = 1
innodb_file_format = Barracuda
innodb_file_format_max = Barracuda
innodb_file_format_check = ON
query_cache_size = 0
query_cache_type = 0


[mysqldump]
quick
max_allowed_packet = 64M

1.3.2、在/tmp目录下编写install.sh安装脚本

#!/bin/sh

yum -y install  gcc gcc-c++ autoconf automake zlib* fiex* libxml* ncurses-devel libmcrypt* libtool-ltdl-devel*
yum -y install make cmake  libaio libaio-devel autoconf bzr bison  libtool perl
yum -y install gcc gcc-c++ ncurses-devel perl libncurses* libaio-devel

userdel db-user
rm -rf /usr/local/mysql
rm -rf /data/mysql

groupadd db-user

useradd  -M  -r -s /sbin/nologin -g db-user db-user

mkdir -p /data/mysql

cd /tmp

tar -xzvf mysql-5.7.26-linux-glibc2.12-x86_64.tar.gz

sleep 1

cp -a  mysql-5.7.26-linux-glibc2.12-x86_64  /usr/local/mysql

echo "PATH=/usr/local/mysql/bin:/usr/local/mysql/lib:\$PATH">>/etc/profile
echo "export PATH">>/etc/profile
source /etc/profile

/usr/local/mysql/bin/mysqld --initialize-insecure --user=db-user --basedir=/usr/local/mysql --datadir=/data/mysql

chown -R db-user. /usr/local/mysql/.
chown -R db-user. /data/mysql/.

cp -a /usr/local/mysql/support-files/mysql.server /etc/init.d/mysqld
chkconfig --add /etc/init.d/mysqld
cp -a /tmp/my.cnf /etc/my.cnf

/etc/init.d/mysqld start


mysql -e "grant all on *.* to 'ztdbuser'@'localhost' identified by '36wPoNGhcH1dA9ig';"
mysql -e "alter user 'root'@'localhost' identified by 'PEIkb3MrTcjKEdyjlm';"

echo "root@localhost password is PEIkb3MrTcjKEdyjlm"

2、下载xxl-job软件

cd /usr/local/src
wget https://github.com/xuxueli/xxl-job/archive/v2.2.0.tar.gz
tar xf v2.2.0.tar.gz -C /usr/local

3、初始化调度数据库

cd /usr/local/xxl-job-2.2.0/doc/db
mysql -u root -pPEIkb3MrTcjKEdyjlm <tables_xxl_job.sql

4、修改配置文件

4.1、修改xxl-job-admin模块配置文件

vim /usr/local/xxl-job-2.2.0/xxl-job-admin/src/main/resources/application.properties
### web
server.port=8080
server.servlet.context-path=/xxl-job-admin

### actuator
management.server.servlet.context-path=/actuator
management.health.mail.enabled=false

### resources
spring.mvc.servlet.load-on-startup=0
spring.mvc.static-path-pattern=/static/**
spring.resources.static-locations=classpath:/static/

### freemarker
spring.freemarker.templateLoaderPath=classpath:/templates/
spring.freemarker.suffix=.ftl
spring.freemarker.charset=UTF-8
spring.freemarker.request-context-attribute=request
spring.freemarker.settings.number_format=0.##########

### mybatis
mybatis.mapper-locations=classpath:/mybatis-mapper/*Mapper.xml
#mybatis.type-aliases-package=com.xxl.job.admin.core.model

### xxl-job, datasource
spring.datasource.url=jdbc:mysql://localhost:63307/xxl_job?useUnicode=true&characterEncoding=UTF-8&autoReconnect=true&serverTimezone=Asia/Shanghai
spring.datasource.username=root
spring.datasource.password=PEIkb3MrTcjKEdyjlm
spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver

### datasource-pool
spring.datasource.type=com.zaxxer.hikari.HikariDataSource
spring.datasource.hikari.minimum-idle=10
spring.datasource.hikari.maximum-pool-size=30
spring.datasource.hikari.auto-commit=true
spring.datasource.hikari.idle-timeout=30000
spring.datasource.hikari.pool-name=HikariCP
spring.datasource.hikari.max-lifetime=900000
spring.datasource.hikari.connection-timeout=10000
spring.datasource.hikari.connection-test-query=SELECT 1

### xxl-job, email
spring.mail.host=smtp.qq.com
spring.mail.port=25
spring.mail.username=xxx@qq.com             #配置报警邮箱账号密码
spring.mail.password=xxx
spring.mail.properties.mail.smtp.auth=true
spring.mail.properties.mail.smtp.starttls.enable=true
spring.mail.properties.mail.smtp.starttls.required=true
spring.mail.properties.mail.smtp.socketFactory.class=javax.net.ssl.SSLSocketFactory

### xxl-job, access token
xxl.job.accessToken=

### xxl-job, i18n (default is zh_CN, and you can choose "zh_CN", "zh_TC" and "en")
xxl.job.i18n=zh_CN

## xxl-job, triggerpool max size
xxl.job.triggerpool.fast.max=200
xxl.job.triggerpool.slow.max=100

### xxl-job, log retention days
xxl.job.logretentiondays=30

4.2、修改xxl-job-admin模块日志设置文件

vim /usr/local/xxl-job-2.2.0/xxl-job-admin/src/main/resources/logback.xml
<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false" scan="true" scanPeriod="1 seconds">

    <contextName>logback</contextName>
    <property name="log.path" value="/usr/local/xxl-job-2.2.0/logs/xxl-job-admin.log"/>

    <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
        <encoder>
            <pattern>%d{HH:mm:ss.SSS} %contextName [%thread] %-5level %logger{36} - %msg%n</pattern>
        </encoder>
    </appender>

    <appender name="file" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <file>${log.path}</file>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>${log.path}.%d{yyyy-MM-dd}.zip</fileNamePattern>
        </rollingPolicy>
        <encoder>
            <pattern>%date %level [%thread] %logger{36} [%file : %line] %msg%n
            </pattern>
        </encoder>
    </appender>

    <root level="info">
        <appender-ref ref="console"/>
        <appender-ref ref="file"/>
    </root>

</configuration>

4.3、修改xxl-job-executor-sample-springboot模块配置文件

vim /usr/local/xxl-job-2.2.0/xxl-job-executor-samples/xxl-job-executor-sample-springboot/src/main/resources/application.properties
# web port
server.port=8081
# no web
#spring.main.web-environment=false

# log config
logging.config=classpath:logback.xml


### xxl-job admin address list, such as "http://address" or "http://address01,http://address02"
xxl.job.admin.addresses=http://127.0.0.1:8080/xxl-job-admin

### xxl-job, access token
xxl.job.accessToken=

### xxl-job executor appname
xxl.job.executor.appname=xxl-job-executor-sample
### xxl-job executor registry-address: default use address to registry , otherwise use ip:port if address is null
xxl.job.executor.address=
### xxl-job executor server-info
xxl.job.executor.ip=                #多网卡时，建议指定一个网卡IP地址绑定host
xxl.job.executor.port=9999
### xxl-job executor log-path
xxl.job.executor.logpath=/usr/local/xxl-job-2.2.0/jobhandler
### xxl-job executor log-retention-days
xxl.job.executor.logretentiondays=30

4.4、修改xxl-job-executor-sample-springboot模块日志设置文件

vim /usr/local/xxl-job-2.2.0/xxl-job-executor-samples/xxl-job-executor-sample-springboot/src/main/resources/logback.xml
<?xml version="1.0" encoding="UTF-8"?>
<configuration debug="false" scan="true" scanPeriod="1 seconds">

    <contextName>logback</contextName>
    <property name="log.path" value="/usr/local/xxl-job-2.2.0/logs/xxl-job-executor-sample-springboot.log"/>

    <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
        <encoder>
            <pattern>%d{HH:mm:ss.SSS} %contextName [%thread] %-5level %logger{36} - %msg%n</pattern>
        </encoder>
    </appender>

    <appender name="file" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <file>${log.path}</file>
        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
            <fileNamePattern>${log.path}.%d{yyyy-MM-dd}.zip</fileNamePattern>
        </rollingPolicy>
        <encoder>
            <pattern>%date %level [%thread] %logger{36} [%file : %line] %msg%n
            </pattern>
        </encoder>
    </appender>

    <root level="info">
        <appender-ref ref="console"/>
        <appender-ref ref="file"/>
    </root>

</configuration>

5、使用mvn编译打包项目、

mkdir /usr/local/xxl-job-2.2.0/logs
mkdir /usr/local/xxl-job-2.2.0/jobhandler
cd /usr/local/xxl-job-2.2.0
mvn package

6、启动服务模块

nohup java -jar xxl-job-admin/target/xxl-job-admin-2.2.0.jar &

7、登录调度页面查看是否正常

http://xxx.xxx.xxx.xxx:8080/xxl-job-admin