hadoop reducer中的空指针异常

【问题标题】：Null pointer exception in hadoop reducerhadoop reducer中的空指针异常
【发布时间】：2014-11-28 15:09:33
【问题描述】：

我正在使用以下代码面对NullPointerException。如果有人可以审查并帮助我完成该计划，那就太好了。

映射器运行良好，但是当我尝试在迭代器处拆分值时，我得到了 NPE。请帮我找出我的错误。我在下面附上了映射器。

Toppermain.java

package TopperPackage;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;


public class TopperMain {
//hadoop jar worcount.jar ars[0] args[1]
public static void main(String[] args) throws Exception {
Job myhadoopJob = new Job();

myhadoopJob.setJarByClass(TopperMain.class);
myhadoopJob.setJobName("Finding topper based on subject");

FileInputFormat.addInputPath(myhadoopJob, new Path(args[0]));
FileOutputFormat.setOutputPath(myhadoopJob, new Path(args[1]));

myhadoopJob.setInputFormatClass(TextInputFormat.class);
myhadoopJob.setOutputFormatClass(TextOutputFormat.class);

myhadoopJob.setMapperClass(TopperMapper.class);
myhadoopJob.setReducerClass(TopperReduce.class);


myhadoopJob.setMapOutputKeyClass(Text.class);
myhadoopJob.setMapOutputValueClass(Text.class);
myhadoopJob.setOutputKeyClass(Text.class);
myhadoopJob.setOutputValueClass(Text.class);

    System.exit(myhadoopJob.waitForCompletion(true) ? 0 : 1);

}

}

TopperMapper.java

package TopperPackage;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
/*Surender,87,60,50,50,80
Raj,80,70,80,85,60
Anten,81,60,50,70,100
Dinesh,60,90,80,80,70
Priya,80,85,91,60,75
*/
public class TopperMapper extends Mapper<LongWritable, Text, Text, Text>

{
String temp,temp2;

protected void map(LongWritable key, Text value,Context context)
    throws IOException, InterruptedException {
String record = value.toString();
String[] parts = record.split(",");
temp=parts[0];

temp2=temp+ "\t" + parts[1];
context.write(new Text("Tamil"),new Text(temp2));
temp2=temp+ "\t" + parts[2];
context.write(new Text("English"),new Text(temp2));

temp2=temp+ "\t" + parts[3];
context.write(new Text("Maths"),new Text(temp2));

temp2=temp+ "\t" + parts[4];
context.write(new Text("Science"),new Text(temp2));

temp2=temp+ "\t" + parts[5];
context.write(new Text("SocialScrience"),new Text(temp2));


       }
}

TopperReduce.java

package TopperPackage;

import java.io.IOException;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;



public class TopperReduce extends Reducer<Text, Text, Text, Text> {
int temp;
private String[] names;
private int[] marks;
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
String top = "";
int count =0,topmark;
marks = null;
String befsplit;
String[] parts=null;
names = null;
for (Text t : values)
{
befsplit= t.toString();
parts = befsplit.split("\t");
names[count]=parts[0];
marks[count]=Integer.parseInt(parts[1]);
count = count+1;

}
topmark=calcTopper(marks);
top=names[topmark]+ "\t"+marks[topmark]  ;
context.write(new Text(key), new Text(top));
}
public int calcTopper(int[] marks)
{
int count=marks.length;
temp=((marks[1]));
int i=0;
for (i=1;i<=(count-2);i++)
{
   if(temp < marks[i+1])
   {
   temp = marks[i+1];

   }

}
return i;
}
}

错误是

cloudera@cloudera-vm:~/Jarfiles$ hadoop jar TopperMain.jar /user/cloudera/inputfiles/topper/topperinput.txt /user/cloudera/outputfiles/topper/
14/08/24 23:17:07 WARN mapred.JobClient: Use GenericOptionsParser for parsing the arguments. Applications should implement Tool for the same.
14/08/24 23:17:08 INFO input.FileInputFormat: Total input paths to process : 1
14/08/24 23:17:09 INFO mapred.JobClient: Running job: job_201408241907_0012
14/08/24 23:17:10 INFO mapred.JobClient:  map 0% reduce 0%
14/08/24 23:17:49 INFO mapred.JobClient:  map 100% reduce 0%
14/08/24 23:18:03 INFO mapred.JobClient: Task Id : attempt_201408241907_0012_r_000000_0, Status : FAILED
java.lang.NullPointerException
    at TopperPackage.TopperReduce.reduce(TopperReduce.java:25)
    at TopperPackage.TopperReduce.reduce(TopperReduce.java:1)
    at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:176)
    at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:571)
    at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:413)
    at org.apache.hadoop.mapred.Child$4.run(Child.java:268)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:396)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1115)
    at org.apache.hadoop.mapred.Child.main(Child.java:262)

attempt_201408241907_0012_r_000000_0: log4j:WARN No appenders could be found for logger (org.apache.hadoop.hdfs.DFSClient).
attempt_201408241907_0012_r_000000_0: log4j:WARN Please initialize the log4j system properly.
14/08/24 23:18:22 INFO mapred.JobClient: Task Id : attempt_201408241907_0012_r_000000_1, Status : FAILED
java.lang.NullPointerException
    at TopperPackage.TopperReduce.reduce(TopperReduce.java:25)
    at TopperPackage.TopperReduce.reduce(TopperReduce.java:1)
    at org.apache.hadoop.mapreduce.Reducer.run(Reducer.java:176)
    at org.apache.hadoop.mapred.ReduceTask.runNewReducer(ReduceTask.java:571)
    at org.apache.hadoop.mapred.ReduceTask.run(ReduceTask.java:413)
    at org.apache.hadoop.mapred.Child$4.run(Child.java:268)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:396)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1115)
    at org.apache.hadoop.mapred.Child.main(Child.java:262)

attempt_201408241907_0012_r_000000_1: log4j:WARN No appenders could be found for logger (org.apache.hadoop.hdfs.DFSClient).
attempt_201408241907_0012_r_000000_1: log4j:WARN Please initialize the log4j system properly.

我从 mapper 获得了预期的输出，但 reducer 在拆分输出并存储在变量中时抛出错误。

映射器输出是

Tamil   Surender    87
English Surender    60
Maths   Surender    50
Science Surender    50
SocialScrience  Surender    80
Tamil   Raj 80
English Raj 70
Maths   Raj 80
Science Raj 85
SocialScrience  Raj 60
Tamil   Anten   81
English Anten   60
Maths   Anten   50
Science Anten   70
SocialScrience  Anten   100
Tamil   Dinesh  60
English Dinesh  90
Maths   Dinesh  80
Science Dinesh  80
SocialScrience  Dinesh  70
Tamil   Priya   80
English Priya   85
Maths   Priya   91
Science Priya   60
SocialScrience  Priya   75

感谢任何指出我的错误的建议。

【问题讨论】：

    标签： hadoop mapreduce


    【解决方案1】：

    错误是由于您将 ma​​rks 和 names 数组初始化为 null 并且没有正确初始化它们。请使用以下减速器类。 

    import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class TopperReduce extends Reducer<Text, Text, Text, Text> {
    int temp;
    private String[] names = new String[10];
    private int[] marks = new int[10];

    public void reduce(Text key, Iterable<Text> values, Context context)
            throws IOException, InterruptedException {
        String top = "";
        int count = 0, topmark;
        String befsplit;
        String[] parts = null;
        for (Text t : values) {
           befsplit = t.toString();
           parts = befsplit.split("\t");
           names[count] = parts[0];
           marks[count] = Integer.parseInt(parts[1]);
           count++;
        }
       topmark = calcTopper(marks);
       top = names[topmark] + "\t" + marks[topmark];
        context.write(new Text(key), new Text(top));
   }

public int calcTopper(int[] marks) {
    int count = marks.length;
    int i = 0;
    int highestMArk = 0;
    int mark = 0;
    int highestMarkIndex = 0;
    for (; i < count; i++) {
        mark = marks[i];
        if (mark > highestMArk) {
            highestMarkIndex = i;

        }
    }
    return highestMarkIndex;
}
}

    【讨论】：

      【解决方案2】：

      您指的是空数组变量部分，因此您收到此错误， 按照我在下面提到的那样更改您的代码它可以工作

       public class TopperReduce extends Reducer<Text, Text, Text, Text> {
int temp;
private String[] names=new String[20];
private int[] marks= new int[20];
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
String top = "";
int count =0,topmark;
for (Text t : values)
{
String befsplit= t.toString();
String[] parts = befsplit.split("\t");
names[count]=parts[0];
marks[count]=Integer.parseInt(parts[1]);
count = count+1;

}
topmark=calcTopper(marks);
top=names[topmark]+ "\t"+marks[topmark]  ;
context.write(new Text(key), new Text(top));
}

      【讨论】：

        猜你喜欢
        • 1970-01-01
        • 2014-07-28
        • 2013-12-19
        • 2016-01-25
        • 2015-07-12
        相关资源
        最近更新 更多
        热门标签
        Java Python linux javascript Mysql C# Docker 算法 前端 SpringBoot Redis Vue spring 设计模式 .net core .net kubernetes c++ 数据库 数据结构 大数据 js 机器学习 微服务 Android Go 程序员 面试 JVM ASP.net core 云原生 人工智能 后端 PHP git CSS golang k8s Nginx Django mybatis 深度学习 多线程 React 架构 devops 爬虫 云计算 Spring Boot LeetCode