diff --git a/src/test/resources/v2readwritetest/partition_csv/partition.csv b/src/test/resources/v2readwritetest/partition_csv/partition.csv deleted file mode 100644 index aead779b..00000000 --- a/src/test/resources/v2readwritetest/partition_csv/partition.csv +++ /dev/null @@ -1,200 +0,0 @@ -col1,col2,col3,col4,col5 -1,fubar_1234,bazbang_78234,barfang_238942,lorem_ipsum_1294 -1,fubar_1235,bazbang_78235,barfang_238943,lorem_ipsum_1295 -1,fubar_1236,bazbang_78236,barfang_238944,lorem_ipsum_1296 -1,fubar_1237,bazbang_78237,barfang_238945,lorem_ipsum_1297 -1,fubar_1238,bazbang_78238,barfang_238946,lorem_ipsum_1298 -1,fubar_1239,bazbang_78239,barfang_238947,lorem_ipsum_1299 -1,fubar_1240,bazbang_78240,barfang_238948,lorem_ipsum_1300 -1,fubar_1241,bazbang_78241,barfang_238949,lorem_ipsum_1301 -2,fubar_1242,bazbang_78242,barfang_238950,lorem_ipsum_1302 -2,fubar_1243,bazbang_78243,barfang_238951,lorem_ipsum_1303 -2,fubar_1244,bazbang_78244,barfang_238952,lorem_ipsum_1304 -2,fubar_1245,bazbang_78245,barfang_238953,lorem_ipsum_1305 -2,fubar_1246,bazbang_78246,barfang_238954,lorem_ipsum_1306 -2,fubar_1247,bazbang_78247,barfang_238955,lorem_ipsum_1307 -2,fubar_1248,bazbang_78248,barfang_238956,lorem_ipsum_1308 -2,fubar_1249,bazbang_78249,barfang_238957,lorem_ipsum_1309 -2,fubar_1250,bazbang_78250,barfang_238958,lorem_ipsum_1310 -2,fubar_1251,bazbang_78251,barfang_238959,lorem_ipsum_1311 -2,fubar_1252,bazbang_78252,barfang_238960,lorem_ipsum_1312 -2,fubar_1253,bazbang_78253,barfang_238961,lorem_ipsum_1313 -2,fubar_1254,bazbang_78254,barfang_238962,lorem_ipsum_1314 -2,fubar_1255,bazbang_78255,barfang_238963,lorem_ipsum_1315 -2,fubar_1256,bazbang_78256,barfang_238964,lorem_ipsum_1316 -2,fubar_1257,bazbang_78257,barfang_238965,lorem_ipsum_1317 -3,fubar_1258,bazbang_78258,barfang_238966,lorem_ipsum_1318 -3,fubar_1259,bazbang_78259,barfang_238967,lorem_ipsum_1319 -3,fubar_1260,bazbang_78260,barfang_238968,lorem_ipsum_1320 -3,fubar_1261,bazbang_78261,barfang_238969,lorem_ipsum_1321 -3,fubar_1262,bazbang_78262,barfang_238970,lorem_ipsum_1322 -3,fubar_1263,bazbang_78263,barfang_238971,lorem_ipsum_1323 -3,fubar_1264,bazbang_78264,barfang_238972,lorem_ipsum_1324 -3,fubar_1265,bazbang_78265,barfang_238973,lorem_ipsum_1325 -3,fubar_1266,bazbang_78266,barfang_238974,lorem_ipsum_1326 -3,fubar_1267,bazbang_78267,barfang_238975,lorem_ipsum_1327 -3,fubar_1268,bazbang_78268,barfang_238976,lorem_ipsum_1328 -4,fubar_1269,bazbang_78269,barfang_238977,lorem_ipsum_1329 -4,fubar_1270,bazbang_78270,barfang_238978,lorem_ipsum_1330 -4,fubar_1271,bazbang_78271,barfang_238979,lorem_ipsum_1331 -4,fubar_1272,bazbang_78272,barfang_238980,lorem_ipsum_1332 -4,fubar_1273,bazbang_78273,barfang_238981,lorem_ipsum_1333 -4,fubar_1274,bazbang_78274,barfang_238982,lorem_ipsum_1334 -4,fubar_1275,bazbang_78275,barfang_238983,lorem_ipsum_1335 -4,fubar_1276,bazbang_78276,barfang_238984,lorem_ipsum_1336 -4,fubar_1277,bazbang_78277,barfang_238985,lorem_ipsum_1337 -4,fubar_1278,bazbang_78278,barfang_238986,lorem_ipsum_1338 -5,fubar_1279,bazbang_78279,barfang_238987,lorem_ipsum_1339 -5,fubar_1280,bazbang_78280,barfang_238988,lorem_ipsum_1340 -5,fubar_1281,bazbang_78281,barfang_238989,lorem_ipsum_1341 -5,fubar_1282,bazbang_78282,barfang_238990,lorem_ipsum_1342 -5,fubar_1283,bazbang_78283,barfang_238991,lorem_ipsum_1343 -5,fubar_1284,bazbang_78284,barfang_238992,lorem_ipsum_1344 -5,fubar_1285,bazbang_78285,barfang_238993,lorem_ipsum_1345 -5,fubar_1286,bazbang_78286,barfang_238994,lorem_ipsum_1346 -5,fubar_1287,bazbang_78287,barfang_238995,lorem_ipsum_1347 -5,fubar_1288,bazbang_78288,barfang_238996,lorem_ipsum_1348 -5,fubar_1289,bazbang_78289,barfang_238997,lorem_ipsum_1349 -6,fubar_1290,bazbang_78290,barfang_238998,lorem_ipsum_1350 -6,fubar_1291,bazbang_78291,barfang_238999,lorem_ipsum_1351 -6,fubar_1292,bazbang_78292,barfang_239000,lorem_ipsum_1352 -6,fubar_1293,bazbang_78293,barfang_239001,lorem_ipsum_1353 -6,fubar_1294,bazbang_78294,barfang_239002,lorem_ipsum_1354 -6,fubar_1295,bazbang_78295,barfang_239003,lorem_ipsum_1355 -6,fubar_1296,bazbang_78296,barfang_239004,lorem_ipsum_1356 -6,fubar_1297,bazbang_78297,barfang_239005,lorem_ipsum_1357 -6,fubar_1298,bazbang_78298,barfang_239006,lorem_ipsum_1358 -6,fubar_1299,bazbang_78299,barfang_239007,lorem_ipsum_1359 -7,fubar_1300,bazbang_78300,barfang_239008,lorem_ipsum_1360 -7,fubar_1301,bazbang_78301,barfang_239009,lorem_ipsum_1361 -7,fubar_1302,bazbang_78302,barfang_239010,lorem_ipsum_1362 -7,fubar_1303,bazbang_78303,barfang_239011,lorem_ipsum_1363 -7,fubar_1304,bazbang_78304,barfang_239012,lorem_ipsum_1364 -7,fubar_1305,bazbang_78305,barfang_239013,lorem_ipsum_1365 -7,fubar_1306,bazbang_78306,barfang_239014,lorem_ipsum_1366 -7,fubar_1307,bazbang_78307,barfang_239015,lorem_ipsum_1367 -7,fubar_1308,bazbang_78308,barfang_239016,lorem_ipsum_1368 -7,fubar_1309,bazbang_78309,barfang_239017,lorem_ipsum_1369 -7,fubar_1310,bazbang_78310,barfang_239018,lorem_ipsum_1370 -8,fubar_1311,bazbang_78311,barfang_239019,lorem_ipsum_1371 -8,fubar_1312,bazbang_78312,barfang_239020,lorem_ipsum_1372 -8,fubar_1313,bazbang_78313,barfang_239021,lorem_ipsum_1373 -8,fubar_1314,bazbang_78314,barfang_239022,lorem_ipsum_1374 -8,fubar_1315,bazbang_78315,barfang_239023,lorem_ipsum_1375 -8,fubar_1316,bazbang_78316,barfang_239024,lorem_ipsum_1376 -8,fubar_1317,bazbang_78317,barfang_239025,lorem_ipsum_1377 -8,fubar_1318,bazbang_78318,barfang_239026,lorem_ipsum_1378 -8,fubar_1319,bazbang_78319,barfang_239027,lorem_ipsum_1379 -8,fubar_1320,bazbang_78320,barfang_239028,lorem_ipsum_1380 -8,fubar_1321,bazbang_78321,barfang_239029,lorem_ipsum_1381 -9,fubar_1322,bazbang_78322,barfang_239030,lorem_ipsum_1382 -9,fubar_1323,bazbang_78323,barfang_239031,lorem_ipsum_1383 -9,fubar_1324,bazbang_78324,barfang_239032,lorem_ipsum_1384 -9,fubar_1325,bazbang_78325,barfang_239033,lorem_ipsum_1385 -9,fubar_1326,bazbang_78326,barfang_239034,lorem_ipsum_1386 -9,fubar_1327,bazbang_78327,barfang_239035,lorem_ipsum_1387 -9,fubar_1328,bazbang_78328,barfang_239036,lorem_ipsum_1388 -9,fubar_1329,bazbang_78329,barfang_239037,lorem_ipsum_1389 -9,fubar_1330,bazbang_78330,barfang_239038,lorem_ipsum_1390 -9,fubar_1331,bazbang_78331,barfang_239039,lorem_ipsum_1391 -10,fubar_1332,bazbang_78332,barfang_239040,lorem_ipsum_1392 -10,fubar_1333,bazbang_78333,barfang_239041,lorem_ipsum_1393 -10,fubar_1334,bazbang_78334,barfang_239042,lorem_ipsum_1394 -10,fubar_1335,bazbang_78335,barfang_239043,lorem_ipsum_1395 -10,fubar_1336,bazbang_78336,barfang_239044,lorem_ipsum_1396 -10,fubar_1337,bazbang_78337,barfang_239045,lorem_ipsum_1397 -10,fubar_1338,bazbang_78338,barfang_239046,lorem_ipsum_1398 -10,fubar_1339,bazbang_78339,barfang_239047,lorem_ipsum_1399 -10,fubar_1340,bazbang_78340,barfang_239048,lorem_ipsum_1400 -10,fubar_1341,bazbang_78341,barfang_239049,lorem_ipsum_1401 -10,fubar_1342,bazbang_78342,barfang_239050,lorem_ipsum_1402 -11,fubar_1343,bazbang_78343,barfang_239051,lorem_ipsum_1403 -11,fubar_1344,bazbang_78344,barfang_239052,lorem_ipsum_1404 -11,fubar_1345,bazbang_78345,barfang_239053,lorem_ipsum_1405 -11,fubar_1346,bazbang_78346,barfang_239054,lorem_ipsum_1406 -11,fubar_1347,bazbang_78347,barfang_239055,lorem_ipsum_1407 -11,fubar_1348,bazbang_78348,barfang_239056,lorem_ipsum_1408 -11,fubar_1349,bazbang_78349,barfang_239057,lorem_ipsum_1409 -11,fubar_1350,bazbang_78350,barfang_239058,lorem_ipsum_1410 -11,fubar_1351,bazbang_78351,barfang_239059,lorem_ipsum_1411 -11,fubar_1352,bazbang_78352,barfang_239060,lorem_ipsum_1412 -11,fubar_1353,bazbang_78353,barfang_239061,lorem_ipsum_1413 -12,fubar_1354,bazbang_78354,barfang_239062,lorem_ipsum_1414 -12,fubar_1355,bazbang_78355,barfang_239063,lorem_ipsum_1415 -12,fubar_1356,bazbang_78356,barfang_239064,lorem_ipsum_1416 -12,fubar_1357,bazbang_78357,barfang_239065,lorem_ipsum_1417 -12,fubar_1358,bazbang_78358,barfang_239066,lorem_ipsum_1418 -12,fubar_1359,bazbang_78359,barfang_239067,lorem_ipsum_1419 -12,fubar_1360,bazbang_78360,barfang_239068,lorem_ipsum_1420 -12,fubar_1361,bazbang_78361,barfang_239069,lorem_ipsum_1421 -12,fubar_1362,bazbang_78362,barfang_239070,lorem_ipsum_1422 -12,fubar_1363,bazbang_78363,barfang_239071,lorem_ipsum_1423 -13,fubar_1364,bazbang_78364,barfang_239072,lorem_ipsum_1424 -13,fubar_1365,bazbang_78365,barfang_239073,lorem_ipsum_1425 -13,fubar_1366,bazbang_78366,barfang_239074,lorem_ipsum_1426 -13,fubar_1367,bazbang_78367,barfang_239075,lorem_ipsum_1427 -13,fubar_1368,bazbang_78368,barfang_239076,lorem_ipsum_1428 -13,fubar_1369,bazbang_78369,barfang_239077,lorem_ipsum_1429 -13,fubar_1370,bazbang_78370,barfang_239078,lorem_ipsum_1430 -13,fubar_1371,bazbang_78371,barfang_239079,lorem_ipsum_1431 -13,fubar_1372,bazbang_78372,barfang_239080,lorem_ipsum_1432 -13,fubar_1373,bazbang_78373,barfang_239081,lorem_ipsum_1433 -13,fubar_1374,bazbang_78374,barfang_239082,lorem_ipsum_1434 -14,fubar_1375,bazbang_78375,barfang_239083,lorem_ipsum_1435 -14,fubar_1376,bazbang_78376,barfang_239084,lorem_ipsum_1436 -14,fubar_1377,bazbang_78377,barfang_239085,lorem_ipsum_1437 -14,fubar_1378,bazbang_78378,barfang_239086,lorem_ipsum_1438 -14,fubar_1379,bazbang_78379,barfang_239087,lorem_ipsum_1439 -14,fubar_1380,bazbang_78380,barfang_239088,lorem_ipsum_1440 -14,fubar_1381,bazbang_78381,barfang_239089,lorem_ipsum_1441 -14,fubar_1382,bazbang_78382,barfang_239090,lorem_ipsum_1442 -14,fubar_1383,bazbang_78383,barfang_239091,lorem_ipsum_1443 -14,fubar_1384,bazbang_78384,barfang_239092,lorem_ipsum_1444 -15,fubar_1385,bazbang_78385,barfang_239093,lorem_ipsum_1445 -15,fubar_1386,bazbang_78386,barfang_239094,lorem_ipsum_1446 -15,fubar_1387,bazbang_78387,barfang_239095,lorem_ipsum_1447 -15,fubar_1388,bazbang_78388,barfang_239096,lorem_ipsum_1448 -15,fubar_1389,bazbang_78389,barfang_239097,lorem_ipsum_1449 -15,fubar_1390,bazbang_78390,barfang_239098,lorem_ipsum_1450 -15,fubar_1391,bazbang_78391,barfang_239099,lorem_ipsum_1451 -15,fubar_1392,bazbang_78392,barfang_239100,lorem_ipsum_1452 -15,fubar_1393,bazbang_78393,barfang_239101,lorem_ipsum_1453 -15,fubar_1394,bazbang_78394,barfang_239102,lorem_ipsum_1454 -15,fubar_1395,bazbang_78395,barfang_239103,lorem_ipsum_1455 -16,fubar_1396,bazbang_78396,barfang_239104,lorem_ipsum_1456 -16,fubar_1397,bazbang_78397,barfang_239105,lorem_ipsum_1457 -16,fubar_1398,bazbang_78398,barfang_239106,lorem_ipsum_1458 -16,fubar_1399,bazbang_78399,barfang_239107,lorem_ipsum_1459 -16,fubar_1400,bazbang_78400,barfang_239108,lorem_ipsum_1460 -16,fubar_1401,bazbang_78401,barfang_239109,lorem_ipsum_1461 -16,fubar_1402,bazbang_78402,barfang_239110,lorem_ipsum_1462 -16,fubar_1403,bazbang_78403,barfang_239111,lorem_ipsum_1463 -16,fubar_1404,bazbang_78404,barfang_239112,lorem_ipsum_1464 -16,fubar_1405,bazbang_78405,barfang_239113,lorem_ipsum_1465 -16,fubar_1406,bazbang_78406,barfang_239114,lorem_ipsum_1466 -17,fubar_1407,bazbang_78407,barfang_239115,lorem_ipsum_1467 -17,fubar_1408,bazbang_78408,barfang_239116,lorem_ipsum_1468 -17,fubar_1409,bazbang_78409,barfang_239117,lorem_ipsum_1469 -17,fubar_1410,bazbang_78410,barfang_239118,lorem_ipsum_1470 -17,fubar_1411,bazbang_78411,barfang_239119,lorem_ipsum_1471 -17,fubar_1412,bazbang_78412,barfang_239120,lorem_ipsum_1472 -17,fubar_1413,bazbang_78413,barfang_239121,lorem_ipsum_1473 -17,fubar_1414,bazbang_78414,barfang_239122,lorem_ipsum_1474 -17,fubar_1415,bazbang_78415,barfang_239123,lorem_ipsum_1475 -17,fubar_1416,bazbang_78416,barfang_239124,lorem_ipsum_1476 -18,fubar_1417,bazbang_78417,barfang_239125,lorem_ipsum_1477 -18,fubar_1418,bazbang_78418,barfang_239126,lorem_ipsum_1478 -18,fubar_1419,bazbang_78419,barfang_239127,lorem_ipsum_1479 -18,fubar_1420,bazbang_78420,barfang_239128,lorem_ipsum_1480 -18,fubar_1421,bazbang_78421,barfang_239129,lorem_ipsum_1481 -18,fubar_1422,bazbang_78422,barfang_239130,lorem_ipsum_1482 -18,fubar_1423,bazbang_78423,barfang_239131,lorem_ipsum_1483 -18,fubar_1424,bazbang_78424,barfang_239132,lorem_ipsum_1484 -18,fubar_1425,bazbang_78425,barfang_239133,lorem_ipsum_1485 -18,fubar_1426,bazbang_78426,barfang_239134,lorem_ipsum_1486 -18,fubar_1427,bazbang_78427,barfang_239135,lorem_ipsum_1487 -19,fubar_1428,bazbang_78428,barfang_239136,lorem_ipsum_1488 -19,fubar_1429,bazbang_78429,barfang_239137,lorem_ipsum_1489 -19,fubar_1430,bazbang_78430,barfang_239138,lorem_ipsum_1490 -19,fubar_1431,bazbang_78431,barfang_239139,lorem_ipsum_1491 -19,fubar_1432,bazbang_78432,barfang_239140,lorem_ipsum_1492 \ No newline at end of file diff --git a/src/test/resources/v2readwritetest/simple_csv/simple.csv b/src/test/resources/v2readwritetest/simple_csv/simple.csv deleted file mode 100644 index 8ea0a820..00000000 --- a/src/test/resources/v2readwritetest/simple_csv/simple.csv +++ /dev/null @@ -1,3 +0,0 @@ -col1;col2;col3 -foo;bar;1 -baz;bang;2 \ No newline at end of file diff --git a/src/test/resources/v2readwritetest/simple_excel/test_simple.xlsx b/src/test/resources/v2readwritetest/simple_excel/test_simple.xlsx deleted file mode 100644 index 66a27245..00000000 Binary files a/src/test/resources/v2readwritetest/simple_excel/test_simple.xlsx and /dev/null differ diff --git a/src/test/scala/com/crealytics/spark/excel/v2/DataFrameWriterApiComplianceSuite.scala b/src/test/scala/com/crealytics/spark/excel/v2/DataFrameWriterApiComplianceSuite.scala index 9eb76834..24fcfd33 100644 --- a/src/test/scala/com/crealytics/spark/excel/v2/DataFrameWriterApiComplianceSuite.scala +++ b/src/test/scala/com/crealytics/spark/excel/v2/DataFrameWriterApiComplianceSuite.scala @@ -22,13 +22,12 @@ import org.scalatest.wordspec.AnyWordSpec class DataFrameWriterApiComplianceSuite extends AnyWordSpec with DataFrameSuiteBase with LocalFileTestingUtilities { - private def readSimpleCsv = { - spark.read - .format("csv") - .option("delimiter", ";") - .option("header", "true") - .option("path", "src/test/resources/v2readwritetest/simple_csv/simple.csv") - .load() + private def simpleDf = { + val data = Seq( + ("foo", "bar", "1"), + ("baz", "bang", "2") + ) + spark.createDataFrame(data).toDF("col1", "col2", "col3") } /** Checks that the excel data files in given folder equal the provided dataframe */ @@ -56,9 +55,9 @@ class DataFrameWriterApiComplianceSuite extends AnyWordSpec with DataFrameSuiteB for (writeMode <- writeModes) { s"write a dataframe to xlsx with ${writeMode.toString}" in withExistingCleanTempDir("v2") { targetDir => // create a df from csv then write as xlsx - val dfCsv = readSimpleCsv + val df = simpleDf - dfCsv.write + df.write .format("excel") .option("path", targetDir) .option("header", value = true) @@ -69,16 +68,16 @@ class DataFrameWriterApiComplianceSuite extends AnyWordSpec with DataFrameSuiteB assert(listOfFiles.nonEmpty, s"expected at least one excel file") // is the result really the same? - assertWrittenExcelData(dfCsv, targetDir) + assertWrittenExcelData(df, targetDir) } s"write a dataframe to xlsx with ${writeMode.toString} (partitioned)" in withExistingCleanTempDir("v2") { targetDir => assume(spark.sparkContext.version >= "3.0.1") // create a df from csv then write as xlsx - val dfCsv = readSimpleCsv + val df = simpleDf - dfCsv.write + df.write .partitionBy("col1") .format("excel") .option("path", targetDir) @@ -96,7 +95,7 @@ class DataFrameWriterApiComplianceSuite extends AnyWordSpec with DataFrameSuiteB } // is the result really the same? - assertWrittenExcelData(dfCsv, targetDir) + assertWrittenExcelData(df, targetDir) } } @@ -107,9 +106,9 @@ class DataFrameWriterApiComplianceSuite extends AnyWordSpec with DataFrameSuiteB assume(spark.sparkContext.version >= "3.0.1") } - val dfCsv = readSimpleCsv + val df = simpleDf - val dfWriter = if (isPartitioned) dfCsv.write else dfCsv.write.partitionBy("col1") + val dfWriter = if (isPartitioned) df.write else df.write.partitionBy("col1") dfWriter .format("excel") @@ -124,9 +123,9 @@ class DataFrameWriterApiComplianceSuite extends AnyWordSpec with DataFrameSuiteB .mode(SaveMode.Append) .save() - val orderedSchemaColumns = dfCsv.schema.fields.map(f => f.name).sorted + val orderedSchemaColumns = df.schema.fields.map(f => f.name).sorted val expectedDf = - dfCsv.union(dfCsv).select(orderedSchemaColumns.head, orderedSchemaColumns.tail.toIndexedSeq: _*) + df.union(df).select(orderedSchemaColumns.head, orderedSchemaColumns.tail.toIndexedSeq: _*) assertWrittenExcelData(expectedDf, targetDir) } diff --git a/src/test/scala/com/crealytics/spark/excel/v2/ManyPartitionReadSuite.scala b/src/test/scala/com/crealytics/spark/excel/v2/ManyPartitionReadSuite.scala index 74d48a5f..da84ba45 100644 --- a/src/test/scala/com/crealytics/spark/excel/v2/ManyPartitionReadSuite.scala +++ b/src/test/scala/com/crealytics/spark/excel/v2/ManyPartitionReadSuite.scala @@ -21,6 +21,7 @@ import org.apache.spark.sql._ import org.apache.spark.sql.functions.col import org.apache.spark.sql.types.IntegerType import org.scalatest.wordspec.AnyWordSpec +import org.apache.spark.sql.types.{StructType, StructField, StringType} class ManyPartitionReadSuite extends AnyWordSpec with DataFrameSuiteBase with LocalFileTestingUtilities { @@ -45,14 +46,36 @@ class ManyPartitionReadSuite extends AnyWordSpec with DataFrameSuiteBase with Lo def createExpected(targetDir: String): DataFrame = { - val dfCsv = spark.read - .format("csv") - .option("delimiter", ",") - .option("header", "true") - .option("path", "src/test/resources/v2readwritetest/partition_csv/partition.csv") - .load() + // Generate data programmatically + val data = (1 to 19).flatMap { col1 => + // Each col1 value has multiple rows (around 10-11 rows each) + val rowsPerPartition = if (col1 == 1) 8 else if (col1 == 2) 16 else 11 + (0 until rowsPerPartition).map { i => + val index = (col1 - 1) * 11 + i + 1234 // Starting from 1234 as in original data + Row( + Integer.valueOf(col1), // Make it nullable Integer + s"fubar_$index", + s"bazbang_${index + 77000}", + s"barfang_${index + 237708}", + s"lorem_ipsum_$index" + ) + } + } + + // Define schema explicitly to match expected nullability + val schema = StructType(Array( + StructField("col1", IntegerType, nullable = true), + StructField("col2", StringType, nullable = true), + StructField("col3", StringType, nullable = true), + StructField("col4", StringType, nullable = true), + StructField("col5", StringType, nullable = true) + )) + + val dfInput = spark.createDataFrame(spark.sparkContext.parallelize(data), schema) + + - val dfFinal = dfCsv.union(dfCsv) + val dfFinal = dfInput.union(dfInput) val dfWriter = dfFinal.write .partitionBy("col1") @@ -64,7 +87,7 @@ class ManyPartitionReadSuite extends AnyWordSpec with DataFrameSuiteBase with Lo dfWriter.save() dfWriter.save() - val orderedSchemaColumns = dfCsv.schema.fields.map(f => f.name).sorted + val orderedSchemaColumns = dfInput.schema.fields.map(f => f.name).sorted dfFinal .union(dfFinal)