spark

spark-default.conf

spark.ui.killEnabled true

spark.serializer org.apache.spark.serializer.KryoSerializer

#spark.executor.extraJavaOptions -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+UseCompressedOops -agentpath:/export/App/yjp-2015-build-15074/bin/linux-x86-64/libyjpagent.so=sampling

#spark.executor.extraJavaOptions -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+UseCompressedOops

#spark.storage.memoryFraction 0.4

spark.default.parallelism 40

spark.kryoserializer.buffer 10m

spark.shuffle.consolidateFiles true

spark.executor.memory 1g

spark.driver.memory 2g

spark.eventLog.enabled true

#spark.eventLog.dir viewfs://nsX/tmp/spark_eventLog

spark.eventLog.dir hdfs://ns1/tmp/spark_eventLog

spark.history.fs.logDirectory hdfs://ns1/tmp/spark_eventLog

spark.yarn.jar hdfs://ns1/user/admin/spark-assembly-1.6.0-hadoop2.6.0.jar

#spark.rdd.compress true

#spark.externalBlockStore.baseDir /data1,/data2

spark.externalBlockStore.baseDir /spark

spark.externalBlockStore.url tachyon://0.0.0.0:19998

#spark.akka.frameSize 10

#spark.scheduler.mode FAIR

#spark.scheduler.allocation.file /export/App/spark-1.4.0-bin-2.6.0/conf/fairscheduler.xml

#spark.executor.userClassPathFirst true

#spark.sql.parquet.binaryAsString true

spark.sql.tungsten.enabled true

spark.sql.sources.partitionDiscovery.enabled true

spark.kryoserializer.buffer.max 512m