The following is a source:
The from pyspark. Mllib. Classification import SVMWithSGD, SVMModel
The from pyspark. Mllib. Regression import LabeledPoint
# the Load and parse the data
Def parsePoint (line) :
[float values=(x) for x in the line. The split (' ')]
Return LabeledPoint (values [0], values [1])
Data=https://bbs.csdn.net/topics/sc.textFile (" HDFS://master: 9000/user/hduser/data/SVM. TXT ")
ParsedData=https://bbs.csdn.net/topics/data.map (parsePoint)
# Build the model
Model=SVMWithSGD. "train" (parsedData, iterations=100)
# Evaluating the model on training data
LabelsAndPreds=parsedData. The map (lambda p: (p.l Abel, model, predict (p.f eatures)))
TrainErr=labelsAndPreds. Filter (lambda (v, p) : v!=p). The count ()/float (parsedData. The count ())
Print (" Training Error="+ STR (trainErr))
-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
The following is the error code:
-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- --
Py4JJavaError Traceback (the most recent call last)
11.
12 # Build the model
---> 13 model=SVMWithSGD. "train" (parsedData, iterations=100)
14
15 # Evaluating the model on training data
/usr/local/spark/python/pyspark mllib/classification. Pyc in train (CLS, data, iterations, step, regParam, miniBatchFraction, initialWeights, regType, intercept, validateData, convergenceTol)
551 bool (intercept), Boolean (validateData), float (convergenceTol))
552
- & gt; 553 return _regression_train_wrapper (train, SVMModel, data, initialWeights)
554
555
/usr/local/spark/python/pyspark mllib/regression. Pyc in _regression_train_wrapper (train_func modelClass, data, initial_weights)
206 def _regression_train_wrapper (train_func modelClass, data, initial_weights) :
207 the from pyspark. Mllib. Classification import LogisticRegressionModel
- & gt; 208 the first=data. The first ()
209 if not isinstance (first, LabeledPoint) :
210 the -raise TypeError (" data should be an RDD of LabeledPoint, but got % s "% type (first))
/usr/local/spark/python/pyspark/RDD. Pyc in first (self)
1359 ValueError: RDD is empty
1360 ", "
"- & gt; 1361 rs=self. Take (1)
1362 if rs:
Return the rs 1363 [0]
/usr/local/spark/python/pyspark/RDD. Pyc take in (self, num)
1341
P=1342 range (partsScanned, min (partsScanned + numPartsToTry totalParts))
- & gt; 1343 res=self. Context. RunJob (self, takeUpToNumLeft, p)
1344
1345 items +=res
/usr/local/spark/python/pyspark/context. Pyc runJob in (self, RDD, partitionFunc, partitions, allowLocal)
963 # SparkContext# runJob.
964 mappedRDD=RDD. MapPartitions (partitionFunc)
- & gt; 965 the port=self. _jvm. PythonRDD. RunJob (self. _jsc. Sc (), mappedRDD. _jrdd, partitions)
966 the return list (_load_from_socket (port, mappedRDD. _jrdd_deserializer))
967
/usr/local/spark/python/lib/py4j - 0.10.4 - SRC. Zip/py4j java_gateway. Py in __call__ (self, * args)
Answer=1131 self. Gateway_client. Send_command (command)
1132 return_value=https://bbs.csdn.net/topics/get_return_value (
- & gt; 1133 answer, self gateway_client, self. Target_id, self. Name)
1134
1135 for temp_arg temp_args in:
/usr/local/spark/python/pyspark/SQL/utils. Pyc in deco (* a, * * kw)
61 def deco (* a, * * kw) :
62 try:
---> 63 return f (a, * * * kw)
64 the except py4j. Protocol. Py4JJavaError as e:
65 s=e.j ava_exception. ToString ()
/usr/local/spark/python/lib/py4j - 0.10.4 - SRC. Zip/py4j/protocol. Py in get_return_value (answer, gateway_client target_id, name)
317 the -raise Py4JJavaError (
318 "An error occurred while calling {0} {1} {2}. \ n".
- & gt; 319 format (target_id, ". ", name), value)
320 else:
321 the -raise Py4JError (
Py4JJavaError: An error occurred while calling z: org. Apache. Spark. API. Python. PythonRDD. RunJob.
: org. Apache. Spark. SparkException: Job aborted due to stage a failure: Task 0 in stage 23.0 failed 4 times, most recent failure: Lost Task in stage 0.3 23.0 (1238, dar 192.168.232.169, executor (2) : org. Apache. The spark. API. Python. PythonException: Traceback (the most recent call last) :
The File "/usr/local/spark/python/lib/pyspark. Zip/pyspark/worker. Py", line 174, the main in
The process ()
The File "/usr/local/spark/python/lib/pyspark. Zip/pyspark/worker. Py", line 169, in the process
nullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnullnull