Hadoop系列-Hive安装

作者: 汤太咸啊 | 来源:发表于2022-03-31 23:24 被阅读0次

前面写了Hadoop默认的组件安装，现在来写一下Hive的安装，这个是Hadoop的一种数据仓库，支持多种文件格式可以直接导入数据库的操作，十分方便。

一、修改yum源
因为是从第一篇重的镜像中直接安装的，我的这个docker镜像是centos6，需要先修改yum源

#备份并修改yum源
cp /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.old
vi /etc/yum.repos.d/CentOS-Base.repo
#修改为如下内容
[base]
name=CentOS-$releasever - Base
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os&infra=$infra
#baseurl=http://mirror.centos.org/centos/$releasever/os/$basearch/
baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos-vault/6.8/os/x86_64/
gpgcheck=1
#gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
gpgkey=https://mirrors.tuna.tsinghua.edu.cn/centos-vault/6.8/os/x86_64/RPM-GPG-KEY-CentOS-6

#released updates
[updates]
name=CentOS-$releasever - Updates
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=updates&infra=$infra
#baseurl=http://mirror.centos.org/centos/$releasever/updates/$basearch/
baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos-vault/6.8/updates/x86_64/
gpgcheck=1
#gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
gpgkey=https://mirrors.tuna.tsinghua.edu.cn/centos-vault/6.8/os/x86_64/RPM-GPG-KEY-CentOS-6

#additional packages that may be useful
[extras]
name=CentOS-$releasever - Extras
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=extras&infra=$infra
#baseurl=http://mirror.centos.org/centos/$releasever/extras/$basearch/
baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos-vault/6.8/extras/x86_64/
gpgcheck=1
#gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
gpgkey=https://mirrors.tuna.tsinghua.edu.cn/centos-vault/6.8/os/x86_64/RPM-GPG-KEY-CentOS-6

#additional packages that extend functionality of existing packages
[centosplus]
name=CentOS-$releasever - Plus
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=centosplus&infra=$infra
#baseurl=http://mirror.centos.org/centos/$releasever/centosplus/$basearch/
baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos-vault/6.8/centosplus/x86_64/
gpgcheck=1
enabled=0
#gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
gpgkey=https://mirrors.tuna.tsinghua.edu.cn/centos-vault/6.8/os/x86_64/RPM-GPG-KEY-CentOS-6

#contrib - packages by Centos Users
[contrib]
name=CentOS-$releasever - Contrib
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=contrib&infra=$infra
#baseurl=http://mirror.centos.org/centos/$releasever/contrib/$basearch/
baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos-vault/6.8/contrib/x86_64/
gpgcheck=1
enabled=0
#gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
gpgkey=https://mirrors.tuna.tsinghua.edu.cn/centos-vault/6.8/os/x86_64/RPM-GPG-KEY-CentOS-6

#修改https验证，增加下面的忽略ssl验证
vi /etc/yum.conf
sslverify=false

二、需要通过yum安装MySQL存储元数据

#先安装MySQL，因为MySQL为Hive提供了元数据的存储
yum install mysql
yum install mysql-server
yum install mysql-devel
#启动MySQL
service mysqld start

#mysql登陆后执行，为本机Hive访问mysql设置密码以及访问权限
grant all on *.* to 'root'@'localhost' identified by 'root';

三、安装JDK

#一般有jdk啊，如果没有手动安装下yum install
yum install java-1.8.0-openjdk.x86_64

四、接下来安装Hive

#到这个位置下载hive包
https://dlcdn.apache.org/hive/hive-1.2.2/

#下载这个版本，另一个2.多的版本不好用，那个依赖了HBASE
apache-hive-1.2.2-bin.tar.gz
#然后cp到docker中
docker cp apache-hive-1.2.2-bin.tar.gz master:/root/

#解压hive，并复制到对应路径下
tar -xzvf apache-hive-1.2.2-bin.tar.gz
mv apache-hive-1.2.2-bin /usr/local/apache-hive-1.2.2-bin

#配置环境变量
vi ~/.bash_profile
#增加以下内容
export HIVE_HOME=/usr/local/apache-hive-1.2.2-bin
export HIVE_CONF_DIR=${HIVE_HOME}/conf 
export JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk.x86_64
#修改完成后，初始化配置
source ~/.bash_profile


#配置中含有hive的一些数据仓库存储位置
#在hadoop中建立该路径并赋权
$HADOOP_HOME/bin/hadoop fs -mkdir -p /user/hive/warehouse
$HADOOP_HOME/bin/hadoop fs -chmod 777 /user/hive/warehouse

$HADOOP_HOME/bin/hadoop fs -mkdir -p /tmp/hive/
$HADOOP_HOME/bin/hadoop fs -chmod 777 /tmp/hive/

#配置hive基础配置
cp hive-default.xml.template hive-site.xml
#修改hive-site.xml
#把所有的${system:java.io.tmpdir}以及${system:user.name}修改为hive路径下，这样后边好查日志之类，具体类似如下修改
#原始
  <property>
    <name>hive.downloaded.resources.dir</name>
    <value>${system:java.io.tmpdir}/${hive.session.id}_resources</value>
    <description>Temporary local directory for added resources in the remote file system.</description>
  </property>
#修改为
  <property>
    <name>hive.downloaded.resources.dir</name>
    <value>/usr/local/apache-hive-1.2.2-bin/tmp/${hive.session.id}_resources</value>
    <description>Temporary local directory for added resources in the remote file system.</description>
  </property>

#原始
  <property>
    <name>hive.server2.logging.operation.log.location</name>
    <value>${system:java.io.tmpdir}/${system:user.name}/operation_logs</value>
    <description>Top level directory where operation logs are stored if logging functionality is enabled</description>
  </property>
#修改为
  <property>
    <name>hive.server2.logging.operation.log.location</name>
    <value>/usr/local/apache-hive-1.2.2-bin/tmp/root/operation_logs</value>
    <description>Top level directory where operation logs are stored if logging functionality is enabled</description>
  </property>

#修改ConnectionURL配置，数据库密码注意，需要与上面安装的一致
  <property>
    <name>javax.jdo.option.ConnectionURL</name>
    <value>jdbc:mysql://127.0.0.1:3306/hive?createDatabaseIfNotExist=true</value>
    <description>JDBC connect string for a JDBC metastore</description>
  </property>
  <property>
    <name>javax.jdo.option.ConnectionDriverName</name>
    <value>com.mysql.jdbc.Driver</value>
    <description>Driver class name for a JDBC metastore</description>
  </property>
  <property>
    <name>javax.jdo.option.ConnectionUserName</name>
    <value>root</value>
    <description>Username to use against metastore database</description>
  </property>
  <property>
    <name>javax.jdo.option.ConnectionPassword</name>
    <value>root</value>
    <description>password to use against metastore database</description>
  </property>
#hive-site.xml修改完币

#复制mysql库到hive路径下，后面初始化时候会使用数据库驱动
cp mysql-connector-java-5.1.46.jar /usr/local/apache-hive-1.2.2-bin/lib/

#修改hive_env配置
cp hive-env.sh.template hive-env.sh
vi hive-env.sh
#增加以下内容
export HADOOP_HOME=/usr/local/hadoop
export HIVE_CONF_DIR=/usr/local/apache-hive-1.2.2-bin/conf
export HIVE_AUX_JARS_PATH=/usr/local/apache-hive-1.2.2-bin/lib

#进入usr/local/apache-hive-1.2.2-bin/执行初始化脚本
./schematool -initSchema -dbType mysql
#初始化成功表明hive配置完成

五、最后咱们一起测试

#编写测试数据文件
vi /usr/local/apache-hive-1.2.2-bin/student.txt
#数据文件注意数据中间不是空格，而是type键，建议vi之后，手动敲一下
001 zhangsan
002 lisi
003 wangwu
004 zhaoliu
005 chenqi
#修改完毕后，可以通过hive进入
hive
#进行hive测试，创建数据库
hive>create database db_hive_test;
hive>use database db_hive_test;
#测试建表
hive>create table student (id int,name string) row format delimited fields terminated by '\t';
#通过刚才建立的文本文件倒入
hive>load data local inpath '/usr/local/apache-hive-1.2.2-bin/test/student.txt' into table db_hive_edu.student;
#执行查询SQL
hive> select * from student;
#输出
OK
1   zhangsan
2   lisi
3   wangwu
4   zhaoliu
5   chenqi
Time taken: 1.872 seconds, Fetched: 5 row(s)

至此，hive安装完成，总结看来怎么感觉就是个MySQL上面套了一层，咱们后边在一起测试看看它到底厉害在哪里。

谢各位的阅读，谢谢您动动手指点赞，万分感谢各位。另外以下是我之前写过的文章，感兴趣的可以点进去继续阅读。

历史文章

Hadoop系列-Hive安装

历史文章

相关文章

网友评论

延伸阅读

深度阅读

栏目导航

热点阅读

小卜java