简介
CDH：全称Cloudera’s Distribution Including Apache Hadoop，是Hadoop众多分支中的一种，由Cloudera维护，基于稳定版本的Apache Hadoop构建。
优点：
- 版本划分清晰
- 版本更新速度快
- 支持Kerberos安全认证
- 文档清晰
- 支持多种安装方式（Cloudera Manager方式）
服务器信息
| IP | hostname |
|---|---|
| 1.1.1.1 | hostname1 |
| 2.2.2.2 | hostname2 |
| 3.3.3.3 | hostname3 |
| 4.4.4.4 | hostname4 |
| 5.5.5.5 | hostname5 |
搭建cdh平台
- 添加用户
adduser cdh
passwd cdh - 赋予权限
chmod u+w /etc/sudoers
vim /etc/sudoers
cdh ALL=(ALL) ALL
chmod u-w /etc/sudoers - 免密登录
ssh-keygen -t rsa
cat id_rsa.pub >> authorized_keys
ssh-copy-id -i ~/.ssh/id_rsa.pub username@xxx.xxx.x.xxx ##将本机的公钥复制到远程机器的authorized_keys文件中.
scp authorized_keys clickhouse@10.101.104.139:/home/clickhouse/.ssh
chmod 600 ~/.ssh/authorized_keys
- 修改主机名
vim /etc/hostname - 修改hosts文件
vim /etc/hosts
#添加如下内容
1.1.1.1 hostname1
2.2.2.2 hostname2
3.3.3.3 hostname3
4.4.4.4 hostname4
5.5.5.5 hostname5
- 关闭防火墙
- 关闭selinux
#查看selinux的状态
/usr/sbin/sestatus -v
vim /etc/sysconfig/selinux
#修改如下行
SELINUX=disabled
- 安装jdk8+
- 安装ntp
CDH要求各个节点时间同步，所以必须安装ntp服务。
#查看ntpd的状态
service ntpd status
#安装时钟服务ntp
yum -y install ntp
#启动ntp服务
service ntpd start
#设置开机自启
chkconfig ntpd on
- 优化交换分区
echo "vm.swappiness = 10" >> /etc/sysctl.conf
sysctl -p
- 禁用透明大页面压缩
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
- 安装mysql5+
mysql用于存储元数据
只需在主节点上安装即可
#安装mysql
https://www.jianshu.com/p/276d59cbc529
#连接mysql
mysql -uroot -p
#创建存储元数据的数据库
create database scm default character set utf8 default collate utf8_general_ci;
create database amon default character set utf8 default collate utf8_general_ci;
create database rman default character set utf8 default collate utf8_general_ci;
create database hue default character set utf8 default collate utf8_general_ci;
create database metastore default character set utf8 default collate utf8_general_ci;
create database sentry default character set utf8 default collate utf8_general_ci;
create database nav default character set utf8 default collate utf8_general_ci;
create database navms default character set utf8 default collate utf8_general_ci;
create database oozie default character set utf8 default collate utf8_general_ci;
#创建用户
create user 'scm'@'%' identified with mysql_native_password by '123456';
create user 'amon'@'%' identified with mysql_native_password by '123456';
create user 'rman'@'%' identified with mysql_native_password by '123456';
create user 'hue'@'%' identified with mysql_native_password by '123456';
create user 'hive'@'%' identified with mysql_native_password by '123456';
create user 'sentry'@'%' identified with mysql_native_password by '123456';
create user 'nav'@'%' identified with mysql_native_password by '123456';
create user 'navms'@'%' identified with mysql_native_password by '123456';
create user 'oozie'@'%' identified with mysql_native_password by '123456';
#赋予权限
grant all on scm.* to 'scm'@'%' with grant option;
grant all on amon.* to 'amon'@'%' with grant option;
grant all on rman.* to 'rman'@'%' with grant option;
grant all on hue.* to 'hue'@'%' with grant option;
grant all on metastore.* to 'hive'@'%' with grant option;
grant all on sentry.* to 'sentry'@'%' with grant option;
grant all on nav.* to 'nav'@'%' with grant option;
grant all on navms.* to 'navms'@'%' with grant option;
#刷新权限
flush privileges;
- 安装mysql-connector
#下载connector
wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.48.tar.gz
#解压
tar -zxvf mysql-connector-java-5.1.48.tar.gz
#
cd mysql-connector-java-5.1.48
#若没有该目录，则创建
mkdir -p /usr/share/java
#复制，改名
cp mysql-connector-java-5.1.48.jar /usr/share/java/mysql-connector-java.jar
#远程复制到集群的其他节点
scp /opt/mysql-connector-java-5.1.48/mysql-connector-java-5.1.48.jar cdh@2.2.2.2:/usr/share/java/mysql-connector-java.jar
- 安装CM（cloudera manager）
cloudera-manager-daemon 与 cloudera-manager-agent 是要在每台机器进行安装的，cloudera-manager-server 只需要在集群中想作为未来管理节点的机器上安装即可 。
#创建目录
mkdir /opt/cloudera-manager
cd /opt/cloudera-manager
#下载cm包
wget http://archive.cloudera.com/cm5/installer/5.14.4/cloudera-manager-installer.bin
wget http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/cloudera-manager.repo
wget http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/5.14.4/RPMS/x86_64/cloudera-manager-agent-5.14.4-1.cm5144.p0.3.el7.x86_64.rpm
wget http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/5.14.4/RPMS/x86_64/cloudera-manager-daemons-5.14.4-1.cm5144.p0.3.el7.x86_64.rpm
wget http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/5.14.4/RPMS/x86_64/cloudera-manager-server-5.14.4-1.cm5144.p0.3.el7.x86_64.rpm
wget http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/5.14.4/RPMS/x86_64/cloudera-manager-server-db-2-5.14.4-1.cm5144.p0.3.el7.x86_64.rpm
wget http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/5.14.4/RPMS/x86_64/enterprise-debuginfo-5.14.4-1.cm5144.p0.3.el7.x86_64.rpm
#复制cloudera-manager.repo到yum仓库
cp -v /opt/cloudera-manager/cloudera-manager.repo /etc/yum.repos.d/
#远程复制两个包到集群的其它节点下
scp cloudera-manager-agent-5.14.4-1.cm5144.p0.3.el7.x86_64.rpm cdh@2.2.2.2:/opt/cloudera-manager/
scp cloudera-manager-daemons-5.14.4-1.cm5144.p0.3.el7.x86_64.rpm root@2.2.2.2:/opt/cloudera-manager/
#在主节点上安装三个包
yum --nogpgcheck -y localinstall cloudera-manager-daemons-*.rpm
yum --nogpgcheck -y localinstall cloudera-manager-agent-*.rpm
yum --nogpgcheck -y localinstall cloudera-manager-server-*.rpm
#在远程节点上安装两个包
ssh -t cdh@2.2.2.2 'yum --nogpgcheck -y localinstall cloudera-manager-daemons-*.rpm'
ssh -t cdh@2.2.2.2 'yum --nogpgcheck -y localinstall cloudera-manager-agent-*.rpm'
#修改agent文件
vim /etc/cloudera-scm-agent/config.ini
#修改内容如下：hostname1为主节点名称
server_host=hostname1
#复制文件到集群的其他节点上
scp /etc/cloudera-scm-agent/config.ini cdh@2.2.2.2:/etc/cloudera-scm-agent/
#主节点创建目录
mkdir /var/lib/cloudera-scm-server
chown cloudera-scm:cloudera-scm /var/lib/cloudera-scm-server
#初始化
/usr/share/cmf/schema/scm_prepare_database.sh -hhostname1 -uroot -p123456 --scm-host hostname1 --force mysql scm scm 123456
- 安装cdh
只在主节点上执行
#创建目录
mkdir -p /opt/cloudera/parcel-repo
cd /opt/cloudera/parcel-repo
#下载cdh包
wget http://archive.cloudera.com/cdh5/parcels/5.14.4/CDH-5.14.4-1.cdh5.14.4.p0.3-el7.parcel
wget http://archive.cloudera.com/cdh5/parcels/5.14.4/CDH-5.14.4-1.cdh5.14.4.p0.3-el7.parcel.sha1
wget http://archive.cloudera.com/cdh5/parcels/5.14.4/manifest.json
#重命名
mv CDH-5.14.4-1.cdh5.14.4.p0.3-el7.parcel.sha1 CDH-5.14.4-1.cdh5.14.4.p0.3-el7.parcel.sha
chown -R cloudera-scm:cloudera-scm /opt/cloudera/parcel-repo
- 启动cm
#启动server
#在主节点上启动
systemctl start cloudera-scm-server
#启动主节点上的agent
systemctl start cloudera-scm-agent
#远程启动集群其它服务器的agent
ssh -t cdh@2.2.2.2 'systemctl start cloudera-scm-agent'
#查看日志
tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log
#查看端口
netstat -tpnl | grep 7180
- 连接cm
在浏览器输入地址：http://1.1.1.1:7180/ - 安装组件
。。。 - 安装spark2
默认cdh安装的spark版本是1.6.0，但是版本太低，可以额外安装spark2，两个版本可以共存。
cd /opt/cloudera
mkdir csd
cd csd
#下载csd包
wget SPARK2_ON_YARN-2.3.0.cloudera3.jar
cd ../parcel-repo
mv manifest.json manifest.json.old
#下载parcels
wget http://archive.cloudera.com/spark2/parcels/2.3.0.cloudera3/SPARK2-2.3.0.cloudera3-1.cdh5.13.3.p0.458809-el7.parcel
wget http://archive.cloudera.com/spark2/parcels/2.3.0.cloudera3/SPARK2-2.3.0.cloudera3-1.cdh5.13.3.p0.458809-el7.parcel.sha1
wget http://archive.cloudera.com/spark2/parcels/2.3.0.cloudera3/manifest.json
#重命名
mv SPARK2-2.3.0.cloudera3-1.cdh5.13.3.p0.458809-el7.parcel.sha1 SPARK2-2.3.0.cloudera3-1.cdh5.13.3.p0.458809-el7.parcel.sha
#重启agent和server
service cloudera-scm-server restart
service cloudera-scm-agent restart
#之后通过cm页面安装spark2
END
网友评论