Support v2 Reading & Writing with Spark Session Properties (#729)

* Support spark session properties * Add spark session property support for writing * Support reading/writing using spark session properties for v2 --------- Co-authored-by: William Phillips <[email protected]>
nightscape · Apr 4, 2023 · ba6c487 · ba6c487
1 parent e9012c3
commit ba6c487
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/src/main/2.4/scala/com/crealytics/spark/excel/v2/ExcelDataSource.scala b/src/main/2.4/scala/com/crealytics/spark/excel/v2/ExcelDataSource.scala
@@ -248,7 +248,7 @@ class ExcelDataSourceReader(
   /* Actual doing schema inferring */
   private def infer(inputPaths: Seq[FileStatus]): StructType = {
     val excelHelper = ExcelHelper(options)
-    val conf = sparkSession.sqlContext.sparkContext.hadoopConfiguration
+    val conf = sparkSession.sessionState.newHadoopConf()
 
     /** Sampling ratio on file level (not row level as in CSV) */
     val paths = {

diff --git a/src/main/3.0_3.1/scala/com/crealytics/spark/excel/v2/ExcelTable.scala b/src/main/3.0_3.1/scala/com/crealytics/spark/excel/v2/ExcelTable.scala
@@ -63,7 +63,7 @@ case class ExcelTable(
   /* Actual doing schema inferring */
   private def infer(sparkSession: SparkSession, inputPaths: Seq[FileStatus], options: ExcelOptions): StructType = {
     val excelHelper = ExcelHelper(options)
-    val conf = sparkSession.sqlContext.sparkContext.hadoopConfiguration
+    val conf = sparkSession.sessionState.newHadoopConf()
 
     /** Sampling ratio on file level (not row level as in CSV) */
     val paths = {

diff --git a/src/main/3.2_and_up/scala/com/crealytics/spark/excel/v2/ExcelTable.scala b/src/main/3.2_and_up/scala/com/crealytics/spark/excel/v2/ExcelTable.scala
@@ -66,7 +66,7 @@ case class ExcelTable(
   /* Actual doing schema inferring */
   private def infer(sparkSession: SparkSession, inputPaths: Seq[FileStatus], options: ExcelOptions): StructType = {
     val excelHelper = ExcelHelper(options)
-    val conf = sparkSession.sqlContext.sparkContext.hadoopConfiguration
+    val conf = sparkSession.sessionState.newHadoopConf()
 
     /* Sampling ratio on file level (not row level as in CSV) */
     val paths = {