1
2
3
4
5
| spark.sessionState.conf.setConfString("spark.sql.streaming.minBatchesToRetain", "5")
logger.info("spark.sql.streaming.minBatchesToRetain:" + spark.sessionState.conf.minBatchesToRetain)
logger.info("spark.sql.shuffle.partitions:" + spark.sparkContext.getConf.get("spark.sql.shuffle.partitions"))
logger.info("spark.sql.streaming.multipleWatermarkPolicy:" + spark.sparkContext.getConf.get("spark.sql.streaming.multipleWatermarkPolicy"))
val logSparkConf = config.getConfig("logSparkConf")
|