Package test.
Import org, apache log4j. Level;
Import org, apache log4j. Logger;
The import org. Apache. Spark. SparkConf;
The import org. Apache. Spark. SparkContext;
The import org. Apache. Spark. RDD. RDD.
Public class ThreadTest {
Public static int port=4050;
Public static void main (String [] args) {
Logger. GetLogger (org. Apache. "spark"). The setLevel (Level. WARN);
Logger. GetLogger (" org. Eclipse. Jetty. Server "). The setLevel (Level. OFF);
//System. SetProperty (" spark. Broadcast. Factory ",
//"org. Apache. Spark. Broadcast. HttpBroadcastFactory");
Thread the t1=new thread ();
Thread t2=new thread ();
The thread t3=new thread ();
T1. Start ();
t2.start();
T3. The start ();
}
Public static () {int g
Return port++;
}
}
The class thread extends thread {
@ Override
Public void the run () {
Int I=ThreadTest. G ();
System. The out. Println (I);
System. SetProperty (" spark. Cores. Max ", "1");
System. SetProperty (" spark. Executor. The memory ", "1 g");
SparkConf conf=new SparkConf (.) setAppName (" test "+ I) setMaster (" spark://master: 7077");
The conf. Set (" spark. UI. The port ", String, the valueOf (I));
SparkContext sc=new SparkContext (conf);
Sc. AddJar ("/home/web/ideaworkspace/test1/out/artifacts/test1_jar/test1. Jar ");
RDD
System. The out. Println (lines. ToJavaRDD (). The count ());
Sc. Stop ();
}
}
4050
4051
4052
Using the Spark 's default log4j profile: org/apache/Spark/log4j - defaults. The properties
SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found the binding in [the jar: file:/home/lib/spark/spark - assembly - 1.1.0 - hadoop2.2.0. Jar!/org/SLF4J/impl/StaticLoggerBinder class]
SLF4J: Found the binding in [the jar: file:/home/spark - assembly - 1.3.1 - hadoop2.3.0. Jar!/org/SLF4J/impl/StaticLoggerBinder class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org. SLF4J. Impl. Log4jLoggerFactory]
15/07/28 19:04:58 INFO Slf4jLogger: Slf4jLogger started
15/07/28 19:04:58 INFO Slf4jLogger: Slf4jLogger started
15/07/28 19:04:58 INFO Slf4jLogger: Slf4jLogger started
15/07/28 19:04:58 INFO Remoting: Starting the Remoting
15/07/28 19:04:58 INFO Remoting: Starting the Remoting
15/07/28 19:04:58 INFO Remoting: Starting the Remoting
15/07/28 19:04:58 INFO Remoting: Remoting started; Listening on addresses: [akka. TCP://sparkDriver @ datanode: 33399]
15/07/28 19:04:58 INFO Remoting: Remoting started; Listening on addresses: [akka. TCP://sparkDriver @ datanode: 58969]
15/07/28 19:04:58 INFO Remoting: Remoting started; Listening on addresses: [akka. TCP://sparkDriver @ datanode: 55465]
15/07/28 19:04:58 INFO Remoting: Remoting now listens on addresses: [akka. TCP://sparkDriver @ datanode: 58969]
15/07/28 19:04:58 INFO Remoting: Remoting now listens on addresses: [akka. TCP://sparkDriver @ datanode: 55465]
15/07/28 19:04:58 INFO Remoting: Remoting now listens on addresses: [akka. TCP://sparkDriver @ datanode: 33399]
15/07/28 19:05:00 WARN NativeCodeLoader: Unable to load the native - hadoop library for your platform... Using the builtin - Java classes where applicable
15/07/28 19:05:04 INFO FileInputFormat: Total input paths to the process: 1
15/07/28 19:05:04 INFO FileInputFormat: Total input paths to the process: 1
15/07/28 19:05:04 INFO FileInputFormat: Total input paths to the process: 1
15/07/28 19:05:05 WARN BlockManager: Block broadcast_1 already exists on this machine. The not re - adding it
15/07/28 19:05:05 WARN BlockManager: Block broadcast_1_piece0 already exists on this machine. The not re - adding it
200
15/07/28 19:05:09 INFO RemoteActorRefProvider $RemotingTerminator: Shutting down the remote daemon.
15/07/28 19:05:09 INFO RemoteActorRefProvider $RemotingTerminator: Remote daemon shut down; Proceeding with flushing remote transports.
15/07/28 19:05:09 INFO Remoting: Remoting shut down
15/07/28 19:05:09 INFO RemoteActorRefProvider $RemotingTerminator: Remoting shut down.
15/07/28 19:05:10 WARN TaskSetManager: Lost task in stage 0.0 0.0 (dar 0, the master) : Java. IO. IOException: unexpected exception type
Java. IO. ObjectStreamClass. ThrowMiscException (ObjectStreamClass. Java: 1538)
Java. IO. ObjectStreamClass. InvokeReadObject (ObjectStreamClass. Java: 1025)
Java. IO. ObjectInputStream. ReadSerialData (ObjectInputStream. Java: 1893)
Java. IO. ObjectInputStream. ReadOrdinaryObject (ObjectInputStream. Java: 1798)
Java. IO. ObjectInputStream. ReadObject0 (ObjectInputStream. Java: 1350)
Java. IO. ObjectInputStream. DefaultReadFields (ObjectInputStream. Java: 1990)
Java. IO. ObjectInputStream. ReadSerialData (ObjectInputStream. Java: 1915)
Java. IO. ObjectInputStream. ReadOrdinaryObject (ObjectInputStream. Java: 1798)
Java. IO. ObjectInputStream. ReadObject0 (ObjectInputStream. Java: 1350)
nullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnull