美文网首页
# kylo安装教程

# kylo安装教程

作者: 夜空最亮的9星 | 来源:发表于2018-09-30 14:53 被阅读451次

    kylo安装脚本

    脚本使用视频

    首先确认安装资源情况如下，根据实际情况配置修改all_config.properties中的配置

    ├── activemq
│   └── apache-activemq-5.15.4-bin.tar.gz
├── all_config.properties
├── elasticsearch
│   └── elasticsearch-6.3.1.tar.gz
├── java
│   └── jdk-8u171-linux-x64.tar.gz
├── kylo
│   └── kylo-0.9.1.tar
├── nifi
│   └── nifi-1.6.0-bin.tar.gz
└── setup_kylo.sh

    all_config.properties配置文件详情

    ##########################👇这里不要修改#################################
ACTIVEMQ_INSTALL_VERSION=5.15.4
ACTIVEMQ_INSTALL_HOME=/opt/activemq
ACTIVEMQ_USER=activemq
ACTIVEMQ_GROUP=activemq
ACTIVEMQ_JAVA_HOME=$JAVA_HOME

NIFI_INSTALL_HOME=/opt/nifi
NIFI_USER=nifi
NIFI_GROUP=nifi
NIFI_VERSION=1.6.0
NIFI_DATA=/opt/nifi/data

KYLO_INSTALL_HOME=/opt/kylo
kylo_home_folder="/opt/kylo"

spark_home="/opt/cloudera/parcels/CDH/lib/spark"
validateAndSplitRecords_extraJars="/opt/cloudera/parcels/CDH/lib/hive-hcatalog/share/hcatalog/hive-hcatalog-core.jar"
hadoopConfigurationResources="/etc/hadoop/conf/core-site.xml,/etc/hadoop/conf/hdfs-site.xml"
hive_lib_path="/opt/cloudera/parcels/CDH/lib/hive/lib"

#########################👆上面不要修改##################################

#kylo需要的mysql数据库，用户名，密码
mysql_kylo_db_host="127.0.0.1"
mysql_kylo_db_user="kylo"
mysql_kylo_db_password="kylo"

#kylo 安装主机IP
kylo_local_ip="10.88.88.121"
#kylo UI 用户密码 ,默认用户名是dladmin
dladmin_password="thinkbig"

#hive2 主机IP
hive_server2_host="10.88.88.120"
hive_metastore_datasource_url="10.88.88.120"

#下面两个没有可以不写
hive_metastore_datasource_username=""
hive_metastore_datasource_password=""


hive_service_principal="hive/kylo1.example.cc@KYLO.CC"
# kerberos.hive.kerberosPrincipal=$hive_service_principal
hive_service_kerberos_keytab="/etc/security/keytabs/hive.service.keytab"
# kerberos.hive.keytabLocation=$hive_service_kerberos_keytab

nifi_service_principal="nifi/kylo2.example.cc@KYLO.CC"
# nifi.service.hive_thrift_service.kerberos_principal=$nifi_service_principal

nifi_service_kerberos_keytab="/etc/security/keytabs/nifi.service.keytab"
# nifi.service.hive_thrift_service.kerberos_keytab=nifi_service_kerberos_keytab

nifi_user_principal="nifi/kylo2.example.cc@KYLO.CC"
# nifi.all_processors.kerberos_principal=$nifi_user_principal

nifi_user_kerberos_keytab="/etc/security/keytabs/nifi.service.keytab"
# nifi.all_processors.kerberos_keytab="nifi_user_kerberos_keytab"


# 配置spark.properties需要用到的配置
# kerberos.spark.kerberosPrincipal
kylo_user_principal="kylo@KYLO.CC"
# kerberos.spark.keytabLocation
kylo_user_kerberos_keytab="/etc/security/keytabs/kylo.user.keytab"

    hdfs创建目录

    [root]# su - hdfs

kinit -kt /etc/security/keytabs/hdfs.service.keytab [hdfs_principal_name]

hdfs dfs -mkdir /user/kylo
hdfs dfs -chown kylo:kylo /user/kylo
hdfs dfs -mkdir /user/nifi
hdfs dfs -chown nifi:nifi /user/nifi
hdfs dfs -mkdir /user/dladmin
hdfs dfs -chown dladmin:dladmin /user/dladmin

hdfs dfs -mkdir /etl
hdfs dfs -chown nifi:nifi /etl
hdfs dfs -mkdir /model.db
hdfs dfs -chown nifi:nifi /model.db
hdfs dfs -mkdir /archive
hdfs dfs -chown nifi:nifi /archive
hdfs dfs -mkdir -p /app/warehouse
hdfs dfs -chown nifi:nifi /app/warehouse

    创建用户和组

    在安装nifi,kylo, activemq的主机上创建用户

    useradd -r -m -s /bin/bash nifi
useradd -r -m -s /bin/bash kylo
useradd -r -m -s /bin/bash activemq

groupadd -f kylo
groupadd -f nifi
groupadd -f activemq

    创建kylo,nifi,hive的principal

    [root@kylo1 ~]# mkdir -p /etc/security/keytabs

[root@kylo1 ~]# kadmin.local

kadmin.local: addprinc -randkey hive_service_principal

kadmin.local: addprinc -randkey nifi_service_principal

kadmin.local: addprinc -randkey nifi_user_principal

kadmin.local: addprinc -randkey kylo_user_principal

    导出principal

    [root@kylo1 ~]#  kadmin.local

kadmin.local: xst -norandkey -k /etc/security/keytabs/hive.service.keytab $hive_service_principal

kadmin.local: xst -norandkey -k /etc/security/keytabs/nifi.service.keytab $nifi_service_principal

kadmin.local: xst -norandkey -k /etc/security/keytabs/nifi.user.keytab $nifi_user_principal

kadmin.local: xst -norandkey -k /etc/security/keytabs/kylo.user.keytab $kylo_user_principal

kadmin.local: q

    把导出的principal拷贝到kylo安装的主机上并执行如下操作

    [root@kylo2 ~]# chmod 440 /etc/security/keytabs/hive.service.keytab

[root@kylo2 ~]# chown kylo:kylo /etc/security/keytabs/hive.service.keytab

[root@kylo2 ~]# chown nifi:nifi /etc/security/keytabs/nifi.service.keytab

[root@kylo2 ~]# chmod 440 /etc/security/keytabs/nifi.service.keytab

[root@kylo2 ~]# chown nifi:nifi /etc/security/keytabs/nifi.user.keytab

[root@kylo2 ~]# chmod 440 /etc/security/keytabs/nifi.user.keytab

[root@kylo2 ~]# chown kylo:kylo /etc/security/keytabs/kylo.user.keytab

[root@kylo2 ~]# chmod 440 /etc/security/keytabs/kylo.user.keytab

    脚本功能预览

    image

    安装mysql

    创建kylo用户数据库

    登录mysql创建kylo用户，密码为kylo

    CREATE USER 'kylo'@'%' IDENTIFIED BY 'kylo';

    授权可以远程登录

    GRANT ALL PRIVILEGES ON *.* TO 'kylo'@'%' IDENTIFIED BY 'kylo' WITH GRANT OPTION;

GRANT ALL PRIVILEGES ON *.* TO 'kylo'@'127.0.0.1' IDENTIFIED BY 'kylo' WITH GRANT OPTION;

GRANT ALL PRIVILEGES ON *.* TO 'kylo'@'localhost' IDENTIFIED BY 'kylo' WITH GRANT OPTION;

GRANT ALL PRIVILEGES ON *.* TO 'kylo'@'10.88.88.1' IDENTIFIED BY 'kylo' WITH GRANT OPTION;

FLUSH PRIVILEGES;

    安装JDK8

    配置JAVA_HOME

    安装ActiveMQ

    安装ActiveMQ之前需要配置JAVA_HOME环境变量

    安装eslasticsearch

    安装eslasticsearch 也需要JAVA_HOME环境变量
    默认去下载 eslasticsearch rpm安装包 自动启动，然后执行创建kylo索引，安装之前最好先安装kylo

    service elasticsearch status

service elasticsearch start

    安装kylo

    image

    配置kylo

    询问用户是否启用kerberos

    Would you enable kerberos ? Please enter y/n:

    在启用kerberos的情况下。要给nifi用户对hive的超级管理员权限。并在hdfs上通过setfacl执行如下命令：

    hdfs dfs -setfacl -R -m user:hive:rwx /app/warehouse

    为kylo创建elastechesarch索引

    需要启动elastechearch

    启动/停止kylo

    kylo启动前，需要先启动activemq , elastechearch ,nifi

    [root@kylo2 ~]# kylo-service start
[root@kylo2 ~]# kylo-service stop

    Check the logs for errors.

    /var/log/kylo-services.log
/var/log/kylo-ui/kylo-ui.log
/var/log/kylo-services/kylo-spark-shell.log

    Login to the Kylo UI.

    http://kylo_host:8400
username:dladmin
password:thinkbig

    导入模板

    特别说明请在 kylo和nifi正常启动后执行

    导入模板后，UI上可以看到如下界面

    image

    安装nifi

    需要用户选择是否启用kerberos

    image

    Start/Stop NiFi

    [root@kylo2 ~]# service nifi start
[root@kylo2 ~]# service nifi stop

    Tail the logs to look for errors.

    tail -f /var/log/nifi/nifi-app.log

    nifi UI

    http://nifi_host:8079/nifi/

    更新kylo数据库

    相当于初始化kylo环境。如果已经导入模板，需要删除nifi并重新安装nifi

    安装jce_policy

    依赖JAVA_HOME

    This extension is required to allow encrypted property values in the Kylo configuration files. If you already have a Java 8 installed on the system, you can install the Java Cryptographic Extension by this.

    停止kylo 、 nifi 、删除nifi[可选]

    为恢复kylo环境准备

    最后预览一下kylo界面

    image

    git源码

    相关文章

      网友评论

          本文标题：# kylo安装教程

          本文链接：https://www.haomeiwen.com/subject/kyewoftx.html

          延伸阅读

          深度阅读

          • 您也可以注册成为美文阅读网的作者，发表您的原创作品、分享您的心情！

          栏目导航

          热点阅读

          关于我们|服务条款|联系我们|# kylo安装教程|投稿指南|网站地图|RSS订阅|排版工具|手机版

          提供经典美文摘抄,优美散文欣赏,现代诗歌精选,短篇小说,心情随笔,表白情书范文,故事会在线阅读欣赏

          Copyright © 2014-2023 Haomeiwen.com All Rights Reserved. 好美文阅读网 版权所有

          备案信息：桂公网安备 45052102000051号 · 桂ICP备13007215号-3

          本站所收录作品、热点评论等信息部分来源互联网，目的只是为了系统归纳学习和传递资讯

          所有作品版权归原创作者所有，与本站立场无关，如不慎侵犯了你的权益，请联系我们告知，我们将做删除处理！