Created
January 3, 2017 06:37
-
-
Save 1ambda/e3326107d14ece9a39663cbc56f05756 to your computer and use it in GitHub Desktop.
pyspark mesos docker
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| [root@moby sbin]# ps -ef | |
| UID PID PPID C STIME TTY TIME CMD | |
| root 1 0 0 06:35 ? 00:00:00 /bin/bash /etc/entrypoint.sh bash | |
| root 152 1 0 06:35 ? 00:00:00 mesos-master --ip=0.0.0.0 --work_dir=/var/lib/mesos | |
| root 153 1 0 06:35 ? 00:00:00 mesos-slave --master=0.0.0.0:5050 --work_dir=/var/lib | |
| root 154 1 0 06:35 ? 00:00:00 bash | |
| root 189 154 0 06:36 ? 00:00:00 ps -ef | |
| [root@moby sbin]# ./bin/pyspark --master mesos://127.0.0.1:5050 --packages com.datastax.spark:spark-cassandra-connector_2.10:1.6.2,TargetHolding:pyspark-cassandra:0.3.5 --exclude-packages org.slf4j:slf4j-api | |
| bash: ./bin/pyspark: No such file or directory | |
| [root@moby sbin]# cd .. | |
| [root@moby spark]# ./bin/pyspark --master mesos://127.0.0.1:5050 --packages com.datastax.spark:spark-cassandra-connector_2.10:1.6.2,TargetHolding:pyspark-cassandra:0.3.5 --exclude-packages org.slf4j:slf4j-api | |
| Python 2.6.6 (r266:84292, Aug 18 2016, 15:13:37) | |
| [GCC 4.4.7 20120313 (Red Hat 4.4.7-17)] on linux2 | |
| Type "help", "copyright", "credits" or "license" for more information. | |
| Ivy Default Cache set to: /root/.ivy2/cache | |
| The jars for the packages stored in: /root/.ivy2/jars | |
| :: loading settings :: url = jar:file:/usr/local/spark-1.6.2-bin-hadoop2.3/lib/spark-assembly-1.6.2-hadoop2.3.0.jar!/org/apache/ivy/core/settings/ivysettings.xml | |
| com.datastax.spark#spark-cassandra-connector_2.10 added as a dependency | |
| TargetHolding#pyspark-cassandra added as a dependency | |
| :: resolving dependencies :: org.apache.spark#spark-submit-parent;1.0 | |
| confs: [default] | |
| found com.datastax.spark#spark-cassandra-connector_2.10;1.6.2 in central | |
| found joda-time#joda-time;2.3 in central | |
| found com.twitter#jsr166e;1.1.0 in central | |
| found io.netty#netty-all;4.0.33.Final in central | |
| found org.joda#joda-convert;1.2 in central | |
| found org.scala-lang#scala-reflect;2.10.5 in central | |
| found TargetHolding#pyspark-cassandra;0.3.5 in spark-packages | |
| found com.datastax.spark#spark-cassandra-connector-java_2.10;1.6.0-M1 in central | |
| found org.apache.cassandra#cassandra-clientutil;3.0.2 in central | |
| found com.datastax.cassandra#cassandra-driver-core;3.0.0 in central | |
| found io.netty#netty-handler;4.0.33.Final in central | |
| found io.netty#netty-buffer;4.0.33.Final in central | |
| found io.netty#netty-common;4.0.33.Final in central | |
| found io.netty#netty-transport;4.0.33.Final in central | |
| found io.netty#netty-codec;4.0.33.Final in central | |
| found io.dropwizard.metrics#metrics-core;3.1.2 in central | |
| found org.apache.commons#commons-lang3;3.3.2 in central | |
| found com.google.guava#guava;16.0.1 in central | |
| downloading https://repo1.maven.org/maven2/com/datastax/spark/spark-cassandra-connector_2.10/1.6.2/spark-cassandra-connector_2.10-1.6.2.jar ... | |
| [SUCCESSFUL ] com.datastax.spark#spark-cassandra-connector_2.10;1.6.2!spark-cassandra-connector_2.10.jar (709ms) | |
| downloading http://dl.bintray.com/spark-packages/maven/TargetHolding/pyspark-cassandra/0.3.5/pyspark-cassandra-0.3.5.jar ... | |
| [SUCCESSFUL ] TargetHolding#pyspark-cassandra;0.3.5!pyspark-cassandra.jar (267ms) | |
| downloading https://repo1.maven.org/maven2/joda-time/joda-time/2.3/joda-time-2.3.jar ... | |
| [SUCCESSFUL ] joda-time#joda-time;2.3!joda-time.jar (101ms) | |
| downloading https://repo1.maven.org/maven2/com/twitter/jsr166e/1.1.0/jsr166e-1.1.0.jar ... | |
| [SUCCESSFUL ] com.twitter#jsr166e;1.1.0!jsr166e.jar (73ms) | |
| downloading https://repo1.maven.org/maven2/io/netty/netty-all/4.0.33.Final/netty-all-4.0.33.Final.jar ... | |
| [SUCCESSFUL ] io.netty#netty-all;4.0.33.Final!netty-all.jar (224ms) | |
| downloading https://repo1.maven.org/maven2/org/joda/joda-convert/1.2/joda-convert-1.2.jar ... | |
| [SUCCESSFUL ] org.joda#joda-convert;1.2!joda-convert.jar (120ms) | |
| downloading https://repo1.maven.org/maven2/org/scala-lang/scala-reflect/2.10.5/scala-reflect-2.10.5.jar ... | |
| [SUCCESSFUL ] org.scala-lang#scala-reflect;2.10.5!scala-reflect.jar (290ms) | |
| downloading https://repo1.maven.org/maven2/com/datastax/spark/spark-cassandra-connector-java_2.10/1.6.0-M1/spark-cassandra-connector-java_2.10-1.6.0-M1.jar ... | |
| [SUCCESSFUL ] com.datastax.spark#spark-cassandra-connector-java_2.10;1.6.0-M1!spark-cassandra-connector-java_2.10.jar (73ms) | |
| downloading https://repo1.maven.org/maven2/org/apache/cassandra/cassandra-clientutil/3.0.2/cassandra-clientutil-3.0.2.jar ... | |
| [SUCCESSFUL ] org.apache.cassandra#cassandra-clientutil;3.0.2!cassandra-clientutil.jar (84ms) | |
| downloading https://repo1.maven.org/maven2/com/datastax/cassandra/cassandra-driver-core/3.0.0/cassandra-driver-core-3.0.0.jar ... | |
| [SUCCESSFUL ] com.datastax.cassandra#cassandra-driver-core;3.0.0!cassandra-driver-core.jar(bundle) (136ms) | |
| downloading https://repo1.maven.org/maven2/org/apache/commons/commons-lang3/3.3.2/commons-lang3-3.3.2.jar ... | |
| [SUCCESSFUL ] org.apache.commons#commons-lang3;3.3.2!commons-lang3.jar (92ms) | |
| downloading https://repo1.maven.org/maven2/com/google/guava/guava/16.0.1/guava-16.0.1.jar ... | |
| [SUCCESSFUL ] com.google.guava#guava;16.0.1!guava.jar(bundle) (226ms) | |
| downloading https://repo1.maven.org/maven2/io/netty/netty-handler/4.0.33.Final/netty-handler-4.0.33.Final.jar ... | |
| [SUCCESSFUL ] io.netty#netty-handler;4.0.33.Final!netty-handler.jar (84ms) | |
| downloading https://repo1.maven.org/maven2/io/dropwizard/metrics/metrics-core/3.1.2/metrics-core-3.1.2.jar ... | |
| [SUCCESSFUL ] io.dropwizard.metrics#metrics-core;3.1.2!metrics-core.jar(bundle) (86ms) | |
| downloading https://repo1.maven.org/maven2/io/netty/netty-buffer/4.0.33.Final/netty-buffer-4.0.33.Final.jar ... | |
| [SUCCESSFUL ] io.netty#netty-buffer;4.0.33.Final!netty-buffer.jar (77ms) | |
| downloading https://repo1.maven.org/maven2/io/netty/netty-transport/4.0.33.Final/netty-transport-4.0.33.Final.jar ... | |
| [SUCCESSFUL ] io.netty#netty-transport;4.0.33.Final!netty-transport.jar (86ms) | |
| downloading https://repo1.maven.org/maven2/io/netty/netty-codec/4.0.33.Final/netty-codec-4.0.33.Final.jar ... | |
| [SUCCESSFUL ] io.netty#netty-codec;4.0.33.Final!netty-codec.jar (127ms) | |
| downloading https://repo1.maven.org/maven2/io/netty/netty-common/4.0.33.Final/netty-common-4.0.33.Final.jar ... | |
| [SUCCESSFUL ] io.netty#netty-common;4.0.33.Final!netty-common.jar (136ms) | |
| :: resolution report :: resolve 21898ms :: artifacts dl 3014ms | |
| :: modules in use: | |
| TargetHolding#pyspark-cassandra;0.3.5 from spark-packages in [default] | |
| com.datastax.cassandra#cassandra-driver-core;3.0.0 from central in [default] | |
| com.datastax.spark#spark-cassandra-connector-java_2.10;1.6.0-M1 from central in [default] | |
| com.datastax.spark#spark-cassandra-connector_2.10;1.6.2 from central in [default] | |
| com.google.guava#guava;16.0.1 from central in [default] | |
| com.twitter#jsr166e;1.1.0 from central in [default] | |
| io.dropwizard.metrics#metrics-core;3.1.2 from central in [default] | |
| io.netty#netty-all;4.0.33.Final from central in [default] | |
| io.netty#netty-buffer;4.0.33.Final from central in [default] | |
| io.netty#netty-codec;4.0.33.Final from central in [default] | |
| io.netty#netty-common;4.0.33.Final from central in [default] | |
| io.netty#netty-handler;4.0.33.Final from central in [default] | |
| io.netty#netty-transport;4.0.33.Final from central in [default] | |
| joda-time#joda-time;2.3 from central in [default] | |
| org.apache.cassandra#cassandra-clientutil;3.0.2 from central in [default] | |
| org.apache.commons#commons-lang3;3.3.2 from central in [default] | |
| org.joda#joda-convert;1.2 from central in [default] | |
| org.scala-lang#scala-reflect;2.10.5 from central in [default] | |
| :: evicted modules: | |
| com.datastax.spark#spark-cassandra-connector_2.10;1.6.0-M1 by [com.datastax.spark#spark-cassandra-connector_2.10;1.6.2] in [default] | |
| --------------------------------------------------------------------- | |
| | | modules || artifacts | | |
| | conf | number| search|dwnlded|evicted|| number|dwnlded| | |
| --------------------------------------------------------------------- | |
| | default | 19 | 18 | 18 | 1 || 18 | 18 | | |
| --------------------------------------------------------------------- | |
| :: retrieving :: org.apache.spark#spark-submit-parent | |
| confs: [default] | |
| 18 artifacts copied, 0 already retrieved (17146kB/31ms) | |
| Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties | |
| 17/01/03 06:37:02 INFO SparkContext: Running Spark version 1.6.2 | |
| 17/01/03 06:37:02 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable | |
| 17/01/03 06:37:02 WARN Utils: Your hostname, moby resolves to a loopback address: 127.0.0.1; using 192.168.65.2 instead (on interface eth0) | |
| 17/01/03 06:37:02 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address | |
| 17/01/03 06:37:02 INFO SecurityManager: Changing view acls to: root | |
| 17/01/03 06:37:02 INFO SecurityManager: Changing modify acls to: root | |
| 17/01/03 06:37:02 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root); users with modify permissions: Set(root) | |
| 17/01/03 06:37:02 INFO Utils: Successfully started service 'sparkDriver' on port 46874. | |
| 17/01/03 06:37:03 INFO Slf4jLogger: Slf4jLogger started | |
| 17/01/03 06:37:03 INFO Remoting: Starting remoting | |
| 17/01/03 06:37:03 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkDriverActorSystem@192.168.65.2:43561] | |
| 17/01/03 06:37:03 INFO Utils: Successfully started service 'sparkDriverActorSystem' on port 43561. | |
| 17/01/03 06:37:03 INFO SparkEnv: Registering MapOutputTracker | |
| 17/01/03 06:37:03 INFO SparkEnv: Registering BlockManagerMaster | |
| 17/01/03 06:37:03 INFO DiskBlockManager: Created local directory at /tmp/blockmgr-e2ed3996-6b74-4de3-b093-5ff5ce1204ad | |
| 17/01/03 06:37:03 INFO MemoryStore: MemoryStore started with capacity 511.5 MB | |
| 17/01/03 06:37:03 INFO SparkEnv: Registering OutputCommitCoordinator | |
| 17/01/03 06:37:03 INFO Utils: Successfully started service 'SparkUI' on port 4040. | |
| 17/01/03 06:37:03 INFO SparkUI: Started SparkUI at http://192.168.65.2:4040 | |
| 17/01/03 06:37:03 INFO HttpFileServer: HTTP File server directory is /tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/httpd-ad8472e2-28e0-4f5f-bb7d-8419b95d5e9d | |
| 17/01/03 06:37:03 INFO HttpServer: Starting HTTP Server | |
| 17/01/03 06:37:03 INFO Utils: Successfully started service 'HTTP file server' on port 43024. | |
| 17/01/03 06:37:03 INFO SparkContext: Added JAR file:/root/.ivy2/jars/com.datastax.spark_spark-cassandra-connector_2.10-1.6.2.jar at http://192.168.65.2:43024/jars/com.datastax.spark_spark-cassandra-connector_2.10-1.6.2.jar with timestamp 1483425423688 | |
| 17/01/03 06:37:03 INFO SparkContext: Added JAR file:/root/.ivy2/jars/TargetHolding_pyspark-cassandra-0.3.5.jar at http://192.168.65.2:43024/jars/TargetHolding_pyspark-cassandra-0.3.5.jar with timestamp1483425423689 | |
| 17/01/03 06:37:03 INFO SparkContext: Added JAR file:/root/.ivy2/jars/joda-time_joda-time-2.3.jar at http://192.168.65.2:43024/jars/joda-time_joda-time-2.3.jar with timestamp 1483425423690 | |
| 17/01/03 06:37:03 INFO SparkContext: Added JAR file:/root/.ivy2/jars/com.twitter_jsr166e-1.1.0.jar at http://192.168.65.2:43024/jars/com.twitter_jsr166e-1.1.0.jar with timestamp 1483425423691 | |
| 17/01/03 06:37:03 INFO SparkContext: Added JAR file:/root/.ivy2/jars/io.netty_netty-all-4.0.33.Final.jar at http://192.168.65.2:43024/jars/io.netty_netty-all-4.0.33.Final.jar with timestamp 1483425423694 | |
| 17/01/03 06:37:03 INFO SparkContext: Added JAR file:/root/.ivy2/jars/org.joda_joda-convert-1.2.jar at http://192.168.65.2:43024/jars/org.joda_joda-convert-1.2.jar with timestamp 1483425423695 | |
| 17/01/03 06:37:03 INFO SparkContext: Added JAR file:/root/.ivy2/jars/org.scala-lang_scala-reflect-2.10.5.jar at http://192.168.65.2:43024/jars/org.scala-lang_scala-reflect-2.10.5.jar with timestamp 1483425423699 | |
| 17/01/03 06:37:03 INFO SparkContext: Added JAR file:/root/.ivy2/jars/com.datastax.spark_spark-cassandra-connector-java_2.10-1.6.0-M1.jar at http://192.168.65.2:43024/jars/com.datastax.spark_spark-cassandra-connector-java_2.10-1.6.0-M1.jar with timestamp 1483425423700 | |
| 17/01/03 06:37:03 INFO SparkContext: Added JAR file:/root/.ivy2/jars/org.apache.cassandra_cassandra-clientutil-3.0.2.jar at http://192.168.65.2:43024/jars/org.apache.cassandra_cassandra-clientutil-3.0.2.jar with timestamp 1483425423701 | |
| 17/01/03 06:37:03 INFO SparkContext: Added JAR file:/root/.ivy2/jars/com.datastax.cassandra_cassandra-driver-core-3.0.0.jar at http://192.168.65.2:43024/jars/com.datastax.cassandra_cassandra-driver-core-3.0.0.jar with timestamp 1483425423702 | |
| 17/01/03 06:37:03 INFO SparkContext: Added JAR file:/root/.ivy2/jars/org.apache.commons_commons-lang3-3.3.2.jar at http://192.168.65.2:43024/jars/org.apache.commons_commons-lang3-3.3.2.jar with timestamp 1483425423703 | |
| 17/01/03 06:37:03 INFO SparkContext: Added JAR file:/root/.ivy2/jars/com.google.guava_guava-16.0.1.jar at http://192.168.65.2:43024/jars/com.google.guava_guava-16.0.1.jar with timestamp 1483425423706 | |
| 17/01/03 06:37:03 INFO SparkContext: Added JAR file:/root/.ivy2/jars/io.netty_netty-handler-4.0.33.Final.jar at http://192.168.65.2:43024/jars/io.netty_netty-handler-4.0.33.Final.jar with timestamp 1483425423707 | |
| 17/01/03 06:37:03 INFO SparkContext: Added JAR file:/root/.ivy2/jars/io.dropwizard.metrics_metrics-core-3.1.2.jar at http://192.168.65.2:43024/jars/io.dropwizard.metrics_metrics-core-3.1.2.jar with timestamp 1483425423707 | |
| 17/01/03 06:37:03 INFO SparkContext: Added JAR file:/root/.ivy2/jars/io.netty_netty-buffer-4.0.33.Final.jar at http://192.168.65.2:43024/jars/io.netty_netty-buffer-4.0.33.Final.jar with timestamp 1483425423708 | |
| 17/01/03 06:37:03 INFO SparkContext: Added JAR file:/root/.ivy2/jars/io.netty_netty-transport-4.0.33.Final.jar at http://192.168.65.2:43024/jars/io.netty_netty-transport-4.0.33.Final.jar with timestamp1483425423709 | |
| 17/01/03 06:37:03 INFO SparkContext: Added JAR file:/root/.ivy2/jars/io.netty_netty-codec-4.0.33.Final.jar at http://192.168.65.2:43024/jars/io.netty_netty-codec-4.0.33.Final.jar with timestamp 1483425423709 | |
| 17/01/03 06:37:03 INFO SparkContext: Added JAR file:/root/.ivy2/jars/io.netty_netty-common-4.0.33.Final.jar at http://192.168.65.2:43024/jars/io.netty_netty-common-4.0.33.Final.jar with timestamp 1483425423710 | |
| 17/01/03 06:37:03 INFO Utils: Copying /root/.ivy2/jars/com.datastax.spark_spark-cassandra-connector_2.10-1.6.2.jar to /tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/com.datastax.spark_spark-cassandra-connector_2.10-1.6.2.jar | |
| 17/01/03 06:37:03 INFO SparkContext: Added file file:/root/.ivy2/jars/com.datastax.spark_spark-cassandra-connector_2.10-1.6.2.jar at http://192.168.65.2:43024/files/com.datastax.spark_spark-cassandra-connector_2.10-1.6.2.jar with timestamp 1483425423948 | |
| 17/01/03 06:37:03 INFO Utils: Copying /root/.ivy2/jars/TargetHolding_pyspark-cassandra-0.3.5.jar to /tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/TargetHolding_pyspark-cassandra-0.3.5.jar | |
| 17/01/03 06:37:03 INFO SparkContext: Added file file:/root/.ivy2/jars/TargetHolding_pyspark-cassandra-0.3.5.jar at http://192.168.65.2:43024/files/TargetHolding_pyspark-cassandra-0.3.5.jar with timestamp 1483425423970 | |
| 17/01/03 06:37:03 INFO Utils: Copying /root/.ivy2/jars/joda-time_joda-time-2.3.jar to /tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/joda-time_joda-time-2.3.jar | |
| 17/01/03 06:37:03 INFO SparkContext: Added file file:/root/.ivy2/jars/joda-time_joda-time-2.3.jar athttp://192.168.65.2:43024/files/joda-time_joda-time-2.3.jar with timestamp 1483425423981 | |
| 17/01/03 06:37:03 INFO Utils: Copying /root/.ivy2/jars/com.twitter_jsr166e-1.1.0.jar to /tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/com.twitter_jsr166e-1.1.0.jar | |
| 17/01/03 06:37:03 INFO SparkContext: Added file file:/root/.ivy2/jars/com.twitter_jsr166e-1.1.0.jar at http://192.168.65.2:43024/files/com.twitter_jsr166e-1.1.0.jar with timestamp 1483425423992 | |
| 17/01/03 06:37:03 INFO Utils: Copying /root/.ivy2/jars/io.netty_netty-all-4.0.33.Final.jar to /tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/io.netty_netty-all-4.0.33.Final.jar | |
| 17/01/03 06:37:04 INFO SparkContext: Added file file:/root/.ivy2/jars/io.netty_netty-all-4.0.33.Final.jar at http://192.168.65.2:43024/files/io.netty_netty-all-4.0.33.Final.jar with timestamp 1483425423999 | |
| 17/01/03 06:37:04 INFO Utils: Copying /root/.ivy2/jars/org.joda_joda-convert-1.2.jar to /tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/org.joda_joda-convert-1.2.jar | |
| 17/01/03 06:37:04 INFO SparkContext: Added file file:/root/.ivy2/jars/org.joda_joda-convert-1.2.jar at http://192.168.65.2:43024/files/org.joda_joda-convert-1.2.jar with timestamp 1483425424005 | |
| 17/01/03 06:37:04 INFO Utils: Copying /root/.ivy2/jars/org.scala-lang_scala-reflect-2.10.5.jar to /tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/org.scala-lang_scala-reflect-2.10.5.jar | |
| 17/01/03 06:37:04 INFO SparkContext: Added file file:/root/.ivy2/jars/org.scala-lang_scala-reflect-2.10.5.jar at http://192.168.65.2:43024/files/org.scala-lang_scala-reflect-2.10.5.jar with timestamp 1483425424012 | |
| 17/01/03 06:37:04 INFO Utils: Copying /root/.ivy2/jars/com.datastax.spark_spark-cassandra-connector-java_2.10-1.6.0-M1.jar to /tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/com.datastax.spark_spark-cassandra-connector-java_2.10-1.6.0-M1.jar | |
| 17/01/03 06:37:04 INFO SparkContext: Added file file:/root/.ivy2/jars/com.datastax.spark_spark-cassandra-connector-java_2.10-1.6.0-M1.jar at http://192.168.65.2:43024/files/com.datastax.spark_spark-cassandra-connector-java_2.10-1.6.0-M1.jar with timestamp 1483425424018 | |
| 17/01/03 06:37:04 INFO Utils: Copying /root/.ivy2/jars/org.apache.cassandra_cassandra-clientutil-3.0.2.jar to /tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/org.apache.cassandra_cassandra-clientutil-3.0.2.jar | |
| 17/01/03 06:37:04 INFO SparkContext: Added file file:/root/.ivy2/jars/org.apache.cassandra_cassandra-clientutil-3.0.2.jar at http://192.168.65.2:43024/files/org.apache.cassandra_cassandra-clientutil-3.0.2.jar with timestamp 1483425424028 | |
| 17/01/03 06:37:04 INFO Utils: Copying /root/.ivy2/jars/com.datastax.cassandra_cassandra-driver-core-3.0.0.jar to /tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/com.datastax.cassandra_cassandra-driver-core-3.0.0.jar | |
| 17/01/03 06:37:04 INFO SparkContext: Added file file:/root/.ivy2/jars/com.datastax.cassandra_cassandra-driver-core-3.0.0.jar at http://192.168.65.2:43024/files/com.datastax.cassandra_cassandra-driver-core-3.0.0.jar with timestamp 1483425424033 | |
| 17/01/03 06:37:04 INFO Utils: Copying /root/.ivy2/jars/org.apache.commons_commons-lang3-3.3.2.jar to/tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/org.apache.commons_commons-lang3-3.3.2.jar | |
| 17/01/03 06:37:04 INFO SparkContext: Added file file:/root/.ivy2/jars/org.apache.commons_commons-lang3-3.3.2.jar at http://192.168.65.2:43024/files/org.apache.commons_commons-lang3-3.3.2.jar with timestamp 1483425424037 | |
| 17/01/03 06:37:04 INFO Utils: Copying /root/.ivy2/jars/com.google.guava_guava-16.0.1.jar to /tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/com.google.guava_guava-16.0.1.jar | |
| 17/01/03 06:37:04 INFO SparkContext: Added file file:/root/.ivy2/jars/com.google.guava_guava-16.0.1.jar at http://192.168.65.2:43024/files/com.google.guava_guava-16.0.1.jar with timestamp 1483425424043 | |
| 17/01/03 06:37:04 INFO Utils: Copying /root/.ivy2/jars/io.netty_netty-handler-4.0.33.Final.jar to /tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/io.netty_netty-handler-4.0.33.Final.jar | |
| 17/01/03 06:37:04 INFO SparkContext: Added file file:/root/.ivy2/jars/io.netty_netty-handler-4.0.33.Final.jar at http://192.168.65.2:43024/files/io.netty_netty-handler-4.0.33.Final.jar with timestamp 1483425424049 | |
| 17/01/03 06:37:04 INFO Utils: Copying /root/.ivy2/jars/io.dropwizard.metrics_metrics-core-3.1.2.jar to /tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/io.dropwizard.metrics_metrics-core-3.1.2.jar | |
| 17/01/03 06:37:04 INFO SparkContext: Added file file:/root/.ivy2/jars/io.dropwizard.metrics_metrics-core-3.1.2.jar at http://192.168.65.2:43024/files/io.dropwizard.metrics_metrics-core-3.1.2.jar with timestamp 1483425424053 | |
| 17/01/03 06:37:04 INFO Utils: Copying /root/.ivy2/jars/io.netty_netty-buffer-4.0.33.Final.jar to /tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/io.netty_netty-buffer-4.0.33.Final.jar | |
| 17/01/03 06:37:04 INFO SparkContext: Added file file:/root/.ivy2/jars/io.netty_netty-buffer-4.0.33.Final.jar at http://192.168.65.2:43024/files/io.netty_netty-buffer-4.0.33.Final.jar with timestamp 1483425424056 | |
| 17/01/03 06:37:04 INFO Utils: Copying /root/.ivy2/jars/io.netty_netty-transport-4.0.33.Final.jar to /tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/io.netty_netty-transport-4.0.33.Final.jar | |
| 17/01/03 06:37:04 INFO SparkContext: Added file file:/root/.ivy2/jars/io.netty_netty-transport-4.0.33.Final.jar at http://192.168.65.2:43024/files/io.netty_netty-transport-4.0.33.Final.jar with timestamp 1483425424060 | |
| 17/01/03 06:37:04 INFO Utils: Copying /root/.ivy2/jars/io.netty_netty-codec-4.0.33.Final.jar to /tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/io.netty_netty-codec-4.0.33.Final.jar | |
| 17/01/03 06:37:04 INFO SparkContext: Added file file:/root/.ivy2/jars/io.netty_netty-codec-4.0.33.Final.jar at http://192.168.65.2:43024/files/io.netty_netty-codec-4.0.33.Final.jar with timestamp 1483425424064 | |
| 17/01/03 06:37:04 INFO Utils: Copying /root/.ivy2/jars/io.netty_netty-common-4.0.33.Final.jar to /tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/io.netty_netty-common-4.0.33.Final.jar | |
| 17/01/03 06:37:04 INFO SparkContext: Added file file:/root/.ivy2/jars/io.netty_netty-common-4.0.33.Final.jar at http://192.168.65.2:43024/files/io.netty_netty-common-4.0.33.Final.jar with timestamp 1483425424068 | |
| WARNING: Logging before InitGoogleLogging() is written to STDERR | |
| W0103 06:37:04.282883 257 sched.cpp:1697] | |
| ************************************************** | |
| Scheduler driver bound to loopback interface! Cannot communicate with remote master(s). You might want to set 'LIBPROCESS_IP' environment variable to use a routable IP address. | |
| ************************************************** | |
| I0103 06:37:04.285872 349 sched.cpp:226] Version: 1.0.0 | |
| I0103 06:37:04.288949 342 sched.cpp:330] New master detected at master@127.0.0.1:5050 | |
| I0103 06:37:04.289577 342 sched.cpp:341] No credentials provided. Attempting to register without authentication | |
| I0103 06:37:04.290807 342 sched.cpp:743] Framework registered with dcfbf7e2-8bee-4c03-adda-dd6536a6a1cb-0000 | |
| 17/01/03 06:37:04 INFO CoarseMesosSchedulerBackend: Registered as framework ID dcfbf7e2-8bee-4c03-adda-dd6536a6a1cb-0000 | |
| 17/01/03 06:37:04 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 45177. | |
| 17/01/03 06:37:04 INFO NettyBlockTransferService: Server created on 45177 | |
| 17/01/03 06:37:04 INFO BlockManagerMaster: Trying to register BlockManager | |
| 17/01/03 06:37:04 INFO BlockManagerMasterEndpoint: Registering block manager 192.168.65.2:45177 with511.5 MB RAM, BlockManagerId(driver, 192.168.65.2, 45177) | |
| 17/01/03 06:37:04 INFO BlockManagerMaster: Registered BlockManager | |
| 17/01/03 06:37:04 INFO CoarseMesosSchedulerBackend: SchedulerBackend is ready for scheduling beginning after reached minRegisteredResourcesRatio: 0.0 | |
| Welcome to | |
| ____ __ | |
| / __/__ ___ _____/ /__ | |
| _\ \/ _ \/ _ `/ __/ '_/ | |
| /__ / .__/\_,_/_/ /_/\_\ version 1.6.2 | |
| /_/ | |
| Using Python version 2.6.6 (r266:84292, Aug 18 2016 15:13:37) | |
| SparkContext available as sc, HiveContext available as sqlContext. | |
| >>> import pyspark_cassandra | |
| Traceback (most recent call last): | |
| File "<stdin>", line 1, in <module> | |
| File "/tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/TargetHolding_pyspark-cassandra-0.3.5.jar/pyspark_cassandra/__init__.py", line 24, in <module> | |
| File "/tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/TargetHolding_pyspark-cassandra-0.3.5.jar/pyspark_cassandra/context.py", line 16, in <module> | |
| File "/tmp/spark-df7bc8fa-233f-4124-855b-4a39fa948c1a/userFiles-ab70ffa3-212b-47ee-9611-9c240d3ce899/TargetHolding_pyspark-cassandra-0.3.5.jar/pyspark_cassandra/rdd.py", line 291 | |
| k = Row(**{c: row.__getattr__(c) for c in columns}) | |
| ^ | |
| SyntaxError: invalid syntax | |
| >>> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment