2021-12-28 pyspark指定本地python环境

pyspark测试

export PYSPARK_PYTHON=./xxx.tar.gz/bin/python &&
export PYSPARK_DRIVER_PYTHON=./python_env/xxx/bin/python &&
pyspark
--driver-cores 2
--executor-memory 8g
--conf spark.pyspark.driver.python=./python_env/xxx/bin/python
--conf spark.yarn.appMasterEnv.PYSPARK_PYTHON=./xxx.tar.gz/bin/python
--conf spark.yarn.dist.archives=hdfs://ns_unknown/env/xxx.tar.gz
--conf spark.jars=hdfs://ns_unknown/jar/spark-tensorflow-connector_2.11-1.14.0.jar,hdfs://ns_unknown/jar/tensorflow-hadoop-1.11.0-rc2.jar
--master yarn
--deploy-mode client
--queue root.bdp_jmart_sch_union.bdp_jmart_dkxwh_union.bdp_jmart_dkxwh_formal

堡垒机spark3.0

export SPARK_HOME=/software/servers/xxx/xxxx/spark_3.0
export PATH=PATH