I tried to write a simple unit for Spark join from the tutorial - Apache Spark Unit Testing Part 2 — Spark SQL. Added all dependencies from reference and
"org.apache.spark" %% "spark-core" % "2.4.7" % Test,
"org.apache.spark" %% "spark-core" % "2.4.7" % Test classifier "tests",
"org.scalatest" %% "scalatest" % "3.2.3" % Test,
but can't add extended QueryTest because it involves AnyFunSuite in private class. Now the code looks like
class TestScalaCheck extends AnyFunSuite with SharedSparkSession {
import testImplicits._
test("join - join using") {
val df = Seq(1, 2, 3).map(i => (i, i.toString)).toDF("int", "str")
val df2 = Seq(1, 2, 3).map(i => (i, (i + 1).toString)).toDF("int", "str")
checkAnswer(
df.join(df2, "int"),
Row(1, "1", "2") :: Row(2, "2", "3") :: Row(3, "3", "5") :: Nil)
}
And it generates the exception message:
An exception or error caused a run to abort: org.apache.spark.sql.test.SharedSparkSession.eventually(Lorg/scalatest/concurrent/PatienceConfiguration$Timeout;Lorg/scalatest/concurrent/PatienceConfiguration$Interval;Lscala/Function0;Lorg/scalactic/source/Position;)Ljava/lang/Object;
java.lang.NoSuchMethodError: org.apache.spark.sql.test.SharedSparkSession.eventually(Lorg/scalatest/concurrent/PatienceConfiguration$Timeout;Lorg/scalatest/concurrent/PatienceConfiguration$Interval;Lscala/Function0;Lorg/scalactic/source/Position;)Ljava/lang/Object;
at org.apache.spark.sql.test.SharedSparkSession.afterEach(SharedSparkSession.scala:135)
at org.apache.spark.sql.test.SharedSparkSession.afterEach$(SharedSparkSession.scala:129)
at lesson2.TestScalaCheck.afterEach(TestScalaCheck.scala:14)
at org.scalatest.BeforeAndAfterEach.$anonfun$runTest$1(BeforeAndAfterEach.scala:247)
at org.scalatest.Status.$anonfun$withAfterEffect$1(Status.scala:377)
at org.scalatest.Status.$anonfun$withAfterEffect$1$adapted(Status.scala:373)
at org.scalatest.SucceededStatus$.whenCompleted(Status.scala:462)
at org.scalatest.Status.withAfterEffect(Status.scala:373)
at org.scalatest.Status.withAfterEffect$(Status.scala:371)
at org.scalatest.SucceededStatus$.withAfterEffect(Status.scala:434)
at org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:246)
at org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:227)
at lesson2.TestScalaCheck.runTest(TestScalaCheck.scala:14)
at org.scalatest.funsuite.AnyFunSuiteLike.$anonfun$runTests$1(AnyFunSuiteLike.scala:233)
at org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:413)
at scala.collection.immutable.List.foreach(List.scala:392)
at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401)
at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:396)
at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:475)
at org.scalatest.funsuite.AnyFunSuiteLike.runTests(AnyFunSuiteLike.scala:233)
at org.scalatest.funsuite.AnyFunSuiteLike.runTests$(AnyFunSuiteLike.scala:232)
at org.scalatest.funsuite.AnyFunSuite.runTests(AnyFunSuite.scala:1563)
at org.scalatest.Suite.run(Suite.scala:1112)
at org.scalatest.Suite.run$(Suite.scala:1094)
at org.scalatest.funsuite.AnyFunSuite.org$scalatest$funsuite$AnyFunSuiteLike$$super$run(AnyFunSuite.scala:1563)
at org.scalatest.funsuite.AnyFunSuiteLike.$anonfun$run$1(AnyFunSuiteLike.scala:237)
at org.scalatest.SuperEngine.runImpl(Engine.scala:535)
at org.scalatest.funsuite.AnyFunSuiteLike.run(AnyFunSuiteLike.scala:237)
at org.scalatest.funsuite.AnyFunSuiteLike.run$(AnyFunSuiteLike.scala:236)
at lesson2.TestScalaCheck.org$scalatest$BeforeAndAfterAll$$super$run(TestScalaCheck.scala:14)
at org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213)
at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210)
at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
at lesson2.TestScalaCheck.run(TestScalaCheck.scala:14)
at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:45)
at org.scalatest.tools.Runner$.$anonfun$doRunRunRunDaDoRunRun$13(Runner.scala:1320)
at org.scalatest.tools.Runner$.$anonfun$doRunRunRunDaDoRunRun$13$adapted(Runner.scala:1314)
at scala.collection.immutable.List.foreach(List.scala:392)
at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:1314)
at org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24(Runner.scala:993)
at org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24$adapted(Runner.scala:971)
at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:1480)
at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:971)
at org.scalatest.tools.Runner$.run(Runner.scala:798)
at org.scalatest.tools.Runner.run(Runner.scala)
at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2or3(ScalaTestRunner.java:38)
at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:25)
Could you help me to resolve it?
The original code is - DataFrameJoinSuite.scala
question from:https://stackoverflow.com/questions/65915911/spark-unit-testing-with-sharedsparksession-org-apache-spark-sql-test-sharedspar
