106.JDBC在Kerberos中的使用

106.1 演示环境介绍

• Kerberos和非Kerberos集群CDH版本：5.12.1
• OS为Redhat版本：7.2
• Spark1.6的ThriftServer服务正常

106.2 操作演示

1.环境前置

• 创建Java工程jdbcdemo

  
  
• 添加Maven依赖

    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.6.5</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hive</groupId>
            <artifactId>hive-jdbc</artifactId>
            <version>1.1.0</version>
        </dependency>
    </dependencies>

2.非Kerberos环境

• 启动非Kerberos环境下的Spark ThriftServer服务

[root@cdh04 ~]# cd /opt/cloudera/parcels/CDH/lib/spark/sbin/
[root@cdh04 sbin]# ./stop-thriftserver.sh 
[root@cdh04 sbin]# rm -rf ../logs/*
[root@cdh04 sbin]# export HADOOP_USER_NAME=hive
[root@cdh04 sbin]# ./start-thriftserver.sh --hiveconf hive.server2.thrift.port=10001 \
>   --hiveconf hive.server2.thrift.bind.host=0.0.0.0 \
>   --hiveconf hive.server2.enable.doAs=true

• 在工程目录下新建NoneKBSample.java文件

package com.cloudera.spark1jdbc;
import com.cloudera.utils.JDBCUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
/**
 * package: com.cloudera.sparkjdbc
 * 公众号：碧茂大数据
 */
public class NoneKBSample {
    private static String JDBC_DRIVER = "org.apache.hive.jdbc.HiveDriver";
    private static String CONNECTION_URL ="jdbc:hive2://cdh04.fayson.com:10001/";
    static {
        try {
            Class.forName(JDBC_DRIVER);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    public static void main(String[] args) {
        System.out.println("使用JDBC的方式访问非Kerberos环境下Spark1.6 Thrift Server");
        Connection connection = null;
        ResultSet rs = null;
        PreparedStatement ps = null;
        try {
            connection = DriverManager.getConnection(CONNECTION_URL, "hive", "");
            ps = connection.prepareStatement("select * from test");
            rs = ps.executeQuery();
            while (rs.next()) {
                System.out.println(rs.getInt(1) + "-------" + rs.getString(2));
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            JDBCUtils.disconnect(connection, rs, ps);
        }
    }
}

3.Kerberos环境

• 把CDH集群中的krb5.conf配置文件拷贝至本地开发环境

# Configuration snippets may be placed in this directory as well
includedir /etc/krb5.conf.d/
[logging]
 default = FILE:/var/log/krb5libs.log
 kdc = FILE:/var/log/krb5kdc.log
 admin_server = FILE:/var/log/kadmind.log
[libdefaults]
 dns_lookup_realm = false
 ticket_lifetime = 24h
 renew_lifetime = 7d
 forwardable = true
 rdns = false
 default_realm = FAYSON.COM
 #default_ccache_name = KEYRING:persistent:%{uid}
[realms]
 FAYSON.COM = {
  kdc = cdh01.fayson.com
  admin_server = cdh01.fayson.com
 }
[domain_realm]
 .fayson.com = FAYSON.COM
 fayson.com = FAYSON.COM

• 在集群KDC和Kadmin所在服务，导出一个用于连接Spark ThriftServer服务的keytab文件

[root@cdh01 ~]# kadmin.local
kadmin.local:  xst -norandkey -k fayson.keytab fayson@FAYSON.COM
kadmin.local:  exit

• 启动Spark1.6的ThriftServer服务
  • 在cdh04.fayson.com启动的ThriftServer，使用hive/cdh04.fayson.com@FAYSON.COM账号启动，且在下面的JDBC连接时也需要该账号

./start-thriftserver.sh --hiveconf hive.server2.authentication.kerberos.principal=hive/cdh04.fayson.com@FAYSON.COM \
  --hiveconf hive.server2.authentication.kerberos.keytab=hive.keytab \
  --principal hive/cdh04.fayson.com@FAYSON.COM --keytab hive.keytab \
  --hiveconf hive.server2.thrift.port=10001 \
  --hiveconf hive.server2.thrift.bind.host=0.0.0.0 \
  --hiveconf hive.server2.enable.doAs=true

• 在工程目录下新建KBSample.java文件

package com.cloudera.spark1jdbc;
import com.cloudera.utils.JDBCUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
/**
 * package: com.cloudera.sparkjdbc
 * 公众号：碧茂大数据
 */
public class KBSample {
    private static String JDBC_DRIVER = "org.apache.hive.jdbc.HiveDriver";
    private static String CONNECTION_URL ="jdbc:hive2://cdh04.fayson.com:10001/;principal=hive/cdh04.fayson.com@FAYSON.COM";
    static {
        try {
            Class.forName(JDBC_DRIVER);
        } catch (ClassNotFoundException e) {
            e.printStackTrace();
        }
    }
    public static void main(String[] args) throws Exception {
        System.out.println("通过JDBC连接Kerberos环境下的Spark1.6 Thrift Server");
        //登录Kerberos账号
        System.setProperty("java.security.krb5.conf", "/Users/fayson/Documents/develop/kerberos/krb5.conf");
        Configuration configuration = new Configuration();
        configuration.set("hadoop.security.authentication" , "Kerberos" );
        UserGroupInformation. setConfiguration(configuration);
        UserGroupInformation.loginUserFromKeytab("fayson@FAYSON.COM", "/Users/fayson/Documents/develop/kerberos/fayson.keytab");
        System.out.println(UserGroupInformation.getLoginUser());
        Connection connection = null;
        ResultSet rs = null;
        PreparedStatement ps = null;
        try {
            connection = DriverManager.getConnection(CONNECTION_URL);
            ps = connection.prepareStatement("select * from test");
            rs = ps.executeQuery();
            while (rs.next()) {
                System.out.println(rs.getInt(1) + "----" + rs.getString(2));
            }
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            JDBCUtils.disconnect(connection, rs, ps);
        }
    }
}

4.总结

• 在启用非Kerberos环境下的Spark ThriftServer服务时需要指定用户为hive，否则在执行查询的时候会出现访问HDFS文件权限问题
• 访问Kerberos环境下的Spark ThriftServer需要在运行环境中增加Kerberos的环境

大数据视频推荐：
腾讯课堂
CSDN
大数据语音推荐：
企业级大数据技术应用
大数据机器学习案例之推荐系统
自然语言处理
大数据基础
人工智能：深度学习入门到精通