Always don't understand is how to obtain the parameter getPartition numPartitions,

 public int getPartition (IntWritable key, IntWritable value, int numPartitions) 

Here the numPartitions related to drink? How to set? Not in the Job related Settings (NumReduceTask), how much is this parameter numPartitions default values,
All the concrete example code stick in the following:

 import org, apache hadoop. Conf. Configuration; 
Import org, apache hadoop. Fs. The Path; 
Import org, apache hadoop. IO. IntWritable; 
Import org, apache hadoop. IO. Text; 
Import org, apache hadoop. Graphs. The Job; 
Import org, apache hadoop. Graphs. Mapper; 
Import org, apache hadoop. The graphs. The Partitioner. 
Import org, apache hadoop. Graphs. Reducer; 
Import org, apache hadoop. Graphs. Lib. Input. FileInputFormat; 
Import org, apache hadoop. Graphs. Lib. Output. FileOutputFormat; 
Import org, apache hadoop. Util. GenericOptionsParser; 

Import the Java. IO. IOException; 

/* * 
* Created by dell on 2017/9/25. 

 * @ auther w* 
*/
Public class MySort {

The static final String INPUT_PATH="HDFS://hadoopwang0:9000/test". 
The static final String OUT_PATH="HDFS://hadoopwang0:9000/testout"; 
Public static void main (String [] args) throws IOException, ClassNotFoundException, InterruptedException {


Configuration conf=new Configuration(); 
//String [] otherArgs=new GenericOptionsParser (conf, args). GetRemainingArgs (); 
//if (otherArgs length!=2) {
//System. Err. Println (" the Usage: wordcount & lt; In> " ); 
//System. Exit (2); 
//} 

Job Job=new Job (conf, "MySort"); 
Job. SetJarByClass (MySort. Class); 
Job. SetMapperClass (MyMap. Class); 
Job. SetReducerClass (MyReduce. Class); 
Job. SetPartitionerClass (MyPartition. Class); 
Job. SetOutputKeyClass (IntWritable. Class); 
Job. SetOutputValueClass (IntWritable. Class); 
FileInputFormat. AddInputPath (job, new Path (INPUT_PATH)); 
FileOutputFormat. SetOutputPath (job, new Path (OUT_PATH)); 
System. The exit (job. WaitForCompletion (true)? 1-0); 
} 

//the Map method: the input value into IntWritable type, as the Key of the output, 
Public static class MyMap extends Mapper {
Private static IntWritable data=https://bbs.csdn.net/topics/new IntWritable (); 

@ Override 
Protected void map (Object key, the Text value, Context Context) throws IOException, InterruptedException {
String line=value.toString(); 

Data set (Integer. ParseInt (line)); 
The context. Write (data, new IntWritable (1)); 
} 

} 

//the Reduce method: the input Key is copied to the output value, and then based on the input & lt; The value - list> The number of elements in the decision to the output frequency of the Key 
//global linenum to represent the key places 
Public static class MyReduce extends ReducerPrivate static IntWritable linenum=new IntWritable (1); 

@ Override 
Protected void the reduce (IntWritable key Iterable Values, the Context Context) throws IOException, InterruptedException {
System. The out. Println (" Reducer: "+ key); 
For (IntWritable val: values) {
The context. Write (linenum, key); 
Linenum=new IntWritable (linenum. The get () + 1); 
} 
} 
} 
//custom Partition function: this function according to the maximum value of input data and the number of Partition in graphs framework to obtain the input data according to the 
//in accordance with the size of the boundary of the block, and then returned to the corresponding relations based on the input value and boundary Partiton ID 
Public static class MyPartition extends Partitioner {
Public int getPartition (IntWritable key, IntWritable value, int numPartitions) {
Int Maxnumber=6522; 
Int bound=Maxnumber/numPartitions + 1; 
Int Keynumber=key. The get (); 
for (int i=0; I & lt; NumPartitions; I++) {
If (Keynumber & lt; I bound * & amp; & Keynumber & gt; Bound * (I - 1))={
The return I - 1; 
} 
} 
return -1; 
} 
} 


} 

CodePudding user response:

Thanks for sharing, with a great help for beginners

CodePudding user response:

Thanks for sharing, with a great help for beginners