本文共 15764 字，大约阅读时间需要 52 分钟。
在使用spark连接mysql的过程中报错了，错误如下
08:51:32.495 [main] ERROR - Error loading factory org.apache.calcite.jdbc.CalciteJdbc41Factoryjava.lang.NoClassDefFoundError: org/apache/calcite/linq4j/QueryProvider at java.lang.ClassLoader.defineClass1(Native Method) ~[?:1.8.0_144] at java.lang.ClassLoader.defineClass(ClassLoader.java:763) ~[?:1.8.0_144] at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) ~[?:1.8.0_144] at java.net.URLClassLoader.defineClass(URLClassLoader.java:467) ~[?:1.8.0_144] at java.net.URLClassLoader.access$100(URLClassLoader.java:73) ~[?:1.8.0_144] at java.net.URLClassLoader$1.run(URLClassLoader.java:368) ~[?:1.8.0_144] at java.net.URLClassLoader$1.run(URLClassLoader.java:362) ~[?:1.8.0_144] at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_144] at java.net.URLClassLoader.findClass(URLClassLoader.java:361) ~[?:1.8.0_144] at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[?:1.8.0_144] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335) ~[?:1.8.0_144] at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[?:1.8.0_144] at java.lang.ClassLoader.defineClass1(Native Method) ~[?:1.8.0_144] at java.lang.ClassLoader.defineClass(ClassLoader.java:763) ~[?:1.8.0_144] at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) ~[?:1.8.0_144] at java.net.URLClassLoader.defineClass(URLClassLoader.java:467) ~[?:1.8.0_144] at java.net.URLClassLoader.access$100(URLClassLoader.java:73) ~[?:1.8.0_144] at java.net.URLClassLoader$1.run(URLClassLoader.java:368) ~[?:1.8.0_144] at java.net.URLClassLoader$1.run(URLClassLoader.java:362) ~[?:1.8.0_144] at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_144] at java.net.URLClassLoader.findClass(URLClassLoader.java:361) ~[?:1.8.0_144] at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[?:1.8.0_144] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335) ~[?:1.8.0_144] at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[?:1.8.0_144] at java.lang.ClassLoader.defineClass1(Native Method) ~[?:1.8.0_144] at java.lang.ClassLoader.defineClass(ClassLoader.java:763) ~[?:1.8.0_144] at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) ~[?:1.8.0_144] at java.net.URLClassLoader.defineClass(URLClassLoader.java:467) ~[?:1.8.0_144] at java.net.URLClassLoader.access$100(URLClassLoader.java:73) ~[?:1.8.0_144] at java.net.URLClassLoader$1.run(URLClassLoader.java:368) ~[?:1.8.0_144] at java.net.URLClassLoader$1.run(URLClassLoader.java:362) ~[?:1.8.0_144] at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_144] at java.net.URLClassLoader.findClass(URLClassLoader.java:361) ~[?:1.8.0_144] at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[?:1.8.0_144] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335) ~[?:1.8.0_144] at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[?:1.8.0_144] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_144] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_144] at org.apache.calcite.avatica.UnregisteredDriver.instantiateFactory(UnregisteredDriver.java:115) [calcite-avatica-1.2.0-incubating.jar:1.2.0-incubating] at org.apache.calcite.avatica.UnregisteredDriver.createFactory(UnregisteredDriver.java:74) [calcite-avatica-1.2.0-incubating.jar:1.2.0-incubating] at org.apache.calcite.avatica.UnregisteredDriver.(UnregisteredDriver.java:55) [calcite-avatica-1.2.0-incubating.jar:1.2.0-incubating] at org.apache.calcite.jdbc.Driver. (Driver.java:53) [calcite-core-1.2.0-incubating.jar:1.2.0-incubating] at org.apache.calcite.jdbc.Driver. (Driver.java:49) [calcite-core-1.2.0-incubating.jar:1.2.0-incubating] at java.lang.Class.forName0(Native Method) [?:1.8.0_144] at java.lang.Class.forName(Class.java:348) [?:1.8.0_144] at java.sql.DriverManager.isDriverAllowed(DriverManager.java:556) [?:1.8.0_144] at java.sql.DriverManager.isDriverAllowed(DriverManager.java:548) [?:1.8.0_144] at java.sql.DriverManager.getDrivers(DriverManager.java:446) [?:1.8.0_144] at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$createConnectionFactory$1.apply(JdbcUtils.scala:52) [spark-sql_2.11-2.1.0.jar:2.1.0] at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$createConnectionFactory$1.apply(JdbcUtils.scala:50) [spark-sql_2.11-2.1.0.jar:2.1.0] at org.apache.spark.sql.execution.datasources.jdbc.JDBCRDD$.resolveTable(JDBCRDD.scala:58) [spark-sql_2.11-2.1.0.jar:2.1.0] at org.apache.spark.sql.execution.datasources.jdbc.JDBCRelation. (JDBCRelation.scala:113) [spark-sql_2.11-2.1.0.jar:2.1.0] at org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider.createRelation(JdbcRelationProvider.scala:45) [spark-sql_2.11-2.1.0.jar:2.1.0] at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:330) [spark-sql_2.11-2.1.0.jar:2.1.0] at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:152) [spark-sql_2.11-2.1.0.jar:2.1.0] at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:125) [spark-sql_2.11-2.1.0.jar:2.1.0] at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. ( :45) [scala-library-2.11.8.jar:?] at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. ( :50) [scala-library-2.11.8.jar:?] at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. ( :52) [scala-library-2.11.8.jar:?] at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. ( :54) [scala-library-2.11.8.jar:?] at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw. ( :56) [scala-library-2.11.8.jar:?] at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw. ( :58) [scala-library-2.11.8.jar:?] at $line31.$read$$iw$$iw$$iw$$iw$$iw$$iw. ( :60) [scala-library-2.11.8.jar:?] at $line31.$read$$iw$$iw$$iw$$iw$$iw. ( :62) [scala-library-2.11.8.jar:?] at $line31.$read$$iw$$iw$$iw$$iw. ( :64) [scala-library-2.11.8.jar:?] at $line31.$read$$iw$$iw$$iw. ( :66) [scala-library-2.11.8.jar:?] at $line31.$read$$iw$$iw. ( :68) [scala-library-2.11.8.jar:?] at $line31.$read$$iw. ( :70) [scala-library-2.11.8.jar:?] at $line31.$read. ( :72) [scala-library-2.11.8.jar:?] at $line31.$read$. ( :76) [scala-library-2.11.8.jar:?] at $line31.$read$. ( ) [scala-library-2.11.8.jar:?] at $line31.$eval$.$print$lzycompute( :7) [scala-library-2.11.8.jar:?] at $line31.$eval$.$print( :6) [scala-library-2.11.8.jar:?] at $line31.$eval.$print( ) [scala-library-2.11.8.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_144] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_144] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_144] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_144] at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786) [scala-compiler-2.11.8.jar:?] at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047) [scala-compiler-2.11.8.jar:?] at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638) [scala-compiler-2.11.8.jar:?] at scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637) [scala-compiler-2.11.8.jar:?] at scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31) [scala-reflect-2.11.8.jar:?] at scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19) [scala-reflect-2.11.8.jar:?] at scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637) [scala-compiler-2.11.8.jar:?] at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569) [scala-compiler-2.11.8.jar:?] at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565) [scala-compiler-2.11.8.jar:?] at scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:807) [scala-compiler-2.11.8.jar:?] at scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:825) [scala-compiler-2.11.8.jar:?] at scala.tools.nsc.interpreter.ILoop.interpretStartingWith(ILoop.scala:825) [scala-compiler-2.11.8.jar:?] at scala.tools.nsc.interpreter.ILoop.command(ILoop.scala:681) [scala-compiler-2.11.8.jar:?] at scala.tools.nsc.interpreter.ILoop.processLine(ILoop.scala:395) [scala-compiler-2.11.8.jar:?] at scala.tools.nsc.interpreter.ILoop.loop(ILoop.scala:415) [scala-compiler-2.11.8.jar:?] at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply$mcZ$sp(ILoop.scala:923) [scala-compiler-2.11.8.jar:?] at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909) [scala-compiler-2.11.8.jar:?] at scala.tools.nsc.interpreter.ILoop$$anonfun$process$1.apply(ILoop.scala:909) [scala-compiler-2.11.8.jar:?] at scala.reflect.internal.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:97) [scala-reflect-2.11.8.jar:?] at scala.tools.nsc.interpreter.ILoop.process(ILoop.scala:909) [scala-compiler-2.11.8.jar:?] at org.apache.spark.repl.Main$.doMain(Main.scala:68) [spark-repl_2.11-2.1.0.jar:2.1.0] at org.apache.spark.repl.Main$.main(Main.scala:51) [spark-repl_2.11-2.1.0.jar:2.1.0] at org.apache.spark.repl.Main.main(Main.scala) [spark-repl_2.11-2.1.0.jar:2.1.0] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_144] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_144] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_144] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_144] at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:738) [spark-core_2.11-2.1.0.jar:2.1.0] at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:187) [spark-core_2.11-2.1.0.jar:2.1.0] at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:212) [spark-core_2.11-2.1.0.jar:2.1.0] at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:126) [spark-core_2.11-2.1.0.jar:2.1.0] at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) [spark-core_2.11-2.1.0.jar:2.1.0]Caused by: java.lang.ClassNotFoundException: org.apache.calcite.linq4j.QueryProvider at java.net.URLClassLoader.findClass(URLClassLoader.java:381) ~[?:1.8.0_144] at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[?:1.8.0_144] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335) ~[?:1.8.0_144] at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[?:1.8.0_144] ... 110 morejava.lang.RuntimeException: Error loading factory org.apache.calcite.jdbc.CalciteJdbc41Factory at org.apache.calcite.avatica.UnregisteredDriver.handle(UnregisteredDriver.java:132) at org.apache.calcite.avatica.UnregisteredDriver.instantiateFactory(UnregisteredDriver.java:126) at org.apache.calcite.avatica.UnregisteredDriver.createFactory(UnregisteredDriver.java:74) at org.apache.calcite.avatica.UnregisteredDriver. (UnregisteredDriver.java:55) at org.apache.calcite.jdbc.Driver. (Driver.java:53) at org.apache.calcite.jdbc.Driver. (Driver.java:49) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:348) at java.sql.DriverManager.isDriverAllowed(DriverManager.java:556) at java.sql.DriverManager.isDriverAllowed(DriverManager.java:548) at java.sql.DriverManager.getDrivers(DriverManager.java:446) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$createConnectionFactory$1.apply(JdbcUtils.scala:52) at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$createConnectionFactory$1.apply(JdbcUtils.scala:50) at org.apache.spark.sql.execution.datasources.jdbc.JDBCRDD$.resolveTable(JDBCRDD.scala:58) at org.apache.spark.sql.execution.datasources.jdbc.JDBCRelation. (JDBCRelation.scala:113) at org.apache.spark.sql.execution.datasources.jdbc.JdbcRelationProvider.createRelation(JdbcRelationProvider.scala:45) at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:330) at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:152) at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:125) ... 54 elidedCaused by: java.lang.NoClassDefFoundError: org/apache/calcite/linq4j/QueryProvider at java.lang.ClassLoader.defineClass1(Native Method) at java.lang.ClassLoader.defineClass(ClassLoader.java:763) at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) at java.net.URLClassLoader.defineClass(URLClassLoader.java:467) at java.net.URLClassLoader.access$100(URLClassLoader.java:73) at java.net.URLClassLoader$1.run(URLClassLoader.java:368) at java.net.URLClassLoader$1.run(URLClassLoader.java:362) at java.security.AccessController.doPrivileged(Native Method) at java.net.URLClassLoader.findClass(URLClassLoader.java:361) at java.lang.ClassLoader.loadClass(ClassLoader.java:424) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335) at java.lang.ClassLoader.loadClass(ClassLoader.java:357) at java.lang.ClassLoader.defineClass1(Native Method) at java.lang.ClassLoader.defineClass(ClassLoader.java:763) at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) at java.net.URLClassLoader.defineClass(URLClassLoader.java:467) at java.net.URLClassLoader.access$100(URLClassLoader.java:73) at java.net.URLClassLoader$1.run(URLClassLoader.java:368) at java.net.URLClassLoader$1.run(URLClassLoader.java:362) at java.security.AccessController.doPrivileged(Native Method) at java.net.URLClassLoader.findClass(URLClassLoader.java:361) at java.lang.ClassLoader.loadClass(ClassLoader.java:424) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335) at java.lang.ClassLoader.loadClass(ClassLoader.java:357) at java.lang.ClassLoader.defineClass1(Native Method) at java.lang.ClassLoader.defineClass(ClassLoader.java:763) at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142) at java.net.URLClassLoader.defineClass(URLClassLoader.java:467) at java.net.URLClassLoader.access$100(URLClassLoader.java:73) at java.net.URLClassLoader$1.run(URLClassLoader.java:368) at java.net.URLClassLoader$1.run(URLClassLoader.java:362) at java.security.AccessController.doPrivileged(Native Method) at java.net.URLClassLoader.findClass(URLClassLoader.java:361) at java.lang.ClassLoader.loadClass(ClassLoader.java:424) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335) at java.lang.ClassLoader.loadClass(ClassLoader.java:357) at java.lang.Class.forName0(Native Method) at java.lang.Class.forName(Class.java:264) at org.apache.calcite.avatica.UnregisteredDriver.instantiateFactory(UnregisteredDriver.java:115) ... 71 moreCaused by: java.lang.ClassNotFoundException: org.apache.calcite.linq4j.QueryProvider at java.net.URLClassLoader.findClass(URLClassLoader.java:381) at java.lang.ClassLoader.loadClass(ClassLoader.java:424) at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:335) at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ... 110 more
连接Mysql的代码是没问题的，但是居然会报这个错误，感觉很奇怪，而且去网上搜还搜不到相关资料，难道只有我碰到这个错误？
一开始以为是mysql的连接jar包有问题，但下载到本地用java连接又可以连接得上Mysql。
这时候注意到了错误ClassNotFoundException，一般这个错误要么是包冲突，要么是包缺失，包冲突就比较恶心了，所以先排查看看会不会是包缺失的问题，在Idea中键入两个shift，在搜素框中搜素org.apache.calcite.jdbc.CalciteJdbc41Factory。找到了这个东西，再去依赖包中查看相关jar包，发现calcite相关的有三个jar包，而在spark的jar包路径下calcite相关jar包只有两个，OK，在mven上下载缺失jar包，问题解决。
转载地址：http://rjgia.baihongyu.com/