Spark伪分布式部署

环境准备

java安装、ssh无密码登陆

spark部署

# vim conf/slaves
192.168.219.51

# vim conf/spark-env.sh
export SPARK_MASTER_HOST=192.168.219.51
export SPARK_MASTER_PORT=7077
export SPARK_WORKER_CORES=1
export SPARK_WORKER_INSTANCES=1
export SPARK_WORKER_MEMORY=3g

./sbin/start-all.sh

客户端访问

./bin/spark-shell --master spark://192.168.219.51:7077
./bin/spark-shell --master spark://192.168.219.51:7077 --executor-mempry 2g

http://192.168.219.51:8080

Spark完全分布部署

环境准备

java安装、scala安装、ssh无密码登陆、hadoop部署

spark部署

mkdir -p /data/spark/{local,log,work}

# vim spark-env.sh
export JAVA_HOME=/usr/local/jdk1.8.0_131
export SCALA_HOME=/usr/local/scala-2.12.2
export HADOOP_HOME=/usr/local/hadoop-2.8.0

export LD_LIBRARY_PATH=${HADOOP_HOME}/lib/native/
export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop/

export SPARK_MASTER_IP=wpsmail-server-5
export SPARK_LOCAL_DIRS=/data/spark/local
export SPARK_LOG_DIR=/data/spark/log
export SPARK_WORKER_DIR=/data/spark/work
export SPARK_DRIVER_MEMORY=1G

# vim slaves
# vim spark-defaults.conf

./sbin/start-all.sh
./sbin/stop-all.sh

客户端访问

bin/spark-shell --master spark://192.168.219.51:7077