考试

hostnamectl set-hostname master

hostnamectl set-hostname slave1

hostnamectl set-hostname slave2

172.18.14.238 master

172.18.14.242 slave1

172.18.14.241 slave2

*/30 10-17 * * * /usr/sbin/ntpdate master

master节点

ssh-keygen

ssh-copy-id -i id_rsa.pub master

ssh-copy-id -i id_rsa.pub slave1

ssh-copy-id -i id_rsa.pub slave2

cd /usr/

mkdir java

cd package277

tar -zxvf jdk-8u221-linux-x64.tar.gz -C /usr/java

cd /usr/java/jdk/

pwd

vim /etc/profile

export JAVA_HOME=/usr/java/jdk1.8.0_221

export JRE_HOME=$JAVA_HOME/jre

export CLASSPATH=$JAVA_HOME/lib

export PATH=$PATH:$JAVA_HOME/bin

source /etc/profile

java -version

-------------------------------

#hadoop

export HADOOP_HOME=/usr/hadoop/hadoop-2.7.7

export CLASSPATH=$CLASSPATH:$HADOOP_HOME/lib

export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

echo master >> master

hostnamectl set-hostname hadoop000 && bash

vim /etc/hosts

添加以下内容

172.18.32.104 hadoop000

注：一个节点都要同时写入一个网址，并且IP为各自分配IP。

ssh-keygen

回车三次

cd /root/.ssh

ssh-copy-id -i id_rsa.pub hadoop000

yes 第一次需要输入密码

ssh hadoop000

ssh localhost

格式化HDFS文件系统

hdfs namenode -format

启动Hadoop集群

start-all.sh

开启mysql服务

systemctl start mysqld

schematool -dbType mysql -initSchema;

hive --service metastore

2、先复制一个master会话窗口，如果要退出：quit;

hive

创建数据库 hive

create database hive;

1.

CREATE  TABLE comm.dim_date ( 

date_id string,

week_id string,

week_day string,

day string,

month string,

quarter string,

year string,

is_workday string,

holiday_id string

)

ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'

LOCATION  '/behavior/dim/dim_date'

TBLPROPERTIES ('skip .header .line .count'= '1');

load data local inpath '/root/bigdata/data/dim_date_2023.txt' into table comm.dim_date;

load data local inpath '/root/bigdata/data/dim_date_2023.txt' overwrite into table comm.dim_date;

2.

create table comm.dim_area

(

city string,

province string,

area string

)

ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'

LOCATION  '/behavior/dim/dim_area';

load data local inpath '/root/bigdata/data/dim_area.txt' overwrite into table comm.dim_area;

3.

create table ads_user_pro

as select province,count(1) cnt

from comm.dws_behavior_log

groupo by province;

insert overwrite local directory '/root/bigdata/result/ads_user_pro'

row format delimited fields terminated by ','

select * from comm.ads_user_pro;

4.

create table ads_user_region

as select b.dt,a.area,count(1) cnt

from comm.dim_area a

comm.dws_behavior_log b

where a.province = b.province

groupo by b.dt,a.area;

insert overwrite local directory '/root/bigdata/result/ads_user_region'

row format delimited fields terminated by ','

select * from comm.ads_user_region;

5.

create table ads_user_hour

as select substring(from_utc_timestamp(ts,'Asia/Shanghai'),12,2) ts_hour,count(1) cnt

from dws_behavior_log

group by substring(from_utc_timestamp(ts,'Asia/Shanghai'),12,2);

insert overwrite local directory '/root/bigdata/result/ads_user_hour'

row format delimited fields terminated by ','

select * from comm.ads_user_hour;

6.

....

7.

create table ads_visit_mode

as select url,device_type,count(1) cnt

from dws_behavior_log

group by url,device_type;

insert overwrite local directory '/root/bigdata/result/ads_visit_mode'

row format delimited fields terminated by ','

select * from comm.ads_visit_mode;

8.

create table ads_online_type

as select url,type,count(1) cnt

from dws_behavior_log

group by url,type;

insert overwrite local directory '/root/bigdata/result/ads_online_type'

row format delimited fields terminated by ','

select * from comm.ads_online_type;

9.

create table ads_user_domain

as select split(url,'.')[1] url_domain,count(1) cnt

from dws_behavior_log

group by split(url,'.')[1],type;

insert overwrite local directory '/root/bigdata/result/ads_user_domain'

row format delimited fields terminated by ','

select * from comm.ads_user_domain ;