windows下Eclipse远程调试运行MR程序

package cn.followtry.hadoop.demo.v2.mr;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import cn.followtry.hadoop.demo.hdfs.HDFSOper;
import cn.followtry.hadoop.demo.util.DebugConfUtil;

/**
 * 
 * brief-hadoop-demo/cn.followtry.hadoop.demo.v2.mr.WordCountV2
 * 
 * @author jingzz
 * @since 2016年12月14日 上午10:03:48
 */
public class WordCountV2 {

	private static final Logger LOGGER = LoggerFactory.getLogger(WordCountV2.class);

	public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
		Configuration conf = new Configuration();
		String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
		if (otherArgs == null || otherArgs.length < 2) {
			System.out.println("用法：\n" + "     至少需要两个参数,最后一个为输出目录，其他为输入文件路径");
			System.exit(-1);
		}
		StringBuilder inputPaths = new StringBuilder();
		String outpathDir;
		int len = otherArgs.length - 1;
		for (int i = 0; i < len; i++) {
			inputPaths.append(otherArgs[i]);
			if (i < len - 1) {
				inputPaths.append(",");
			}
		}
		outpathDir = otherArgs[len];
		// 检查输出目录是否存在，存在则直接删除目录
		HDFSOper.rmExistsOutputDir(outpathDir);

		// 只有在当前系统为windows是设置该debug配置
		String osName = System.getProperties().getProperty("os.name").toLowerCase();
		if (osName.contains("windows")) {
			LOGGER.info("运行在windows平台，需要设置配置");
			String jarPath = "d:\\mapreduce.jar";
			DebugConfUtil.setLocalDebugConfiguration(jarPath, conf);
		} else {
			LOGGER.info("运行在{}平台", osName);
		}
		Job job = Job.getInstance(conf, "wordCount v2 demo 2");

		job.setJarByClass(WordCountV2.class);

		job.setMapperClass(WordCountMapV2.class);
		job.setCombinerClass(WordCountReduceV2.class);
		job.setReducerClass(WordCountReduceV2.class);

		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(IntWritable.class);

		FileInputFormat.setInputPaths(job, inputPaths.toString());
		FileOutputFormat.setOutputPath(job, new Path(outpathDir));

		System.exit(job.waitForCompletion(true) ? 0 : 1);
	}
}

package cn.followtry.hadoop.demo.util;

import org.apache.hadoop.conf.Configuration;

/**
 *  brief-hadoop-demo/cn.followtry.hadoop.demo.util.DebugConfUtil
 * @author 
 *		jingzz 
 * @since 
 *		2016年12月15日 下午12:32:02
 */
public class DebugConfUtil {
	
	private DebugConfUtil() {
	}
	
	/**
	 * 设置win本地调试提交MR程序到远程YARN集群上的配置信息
	 * @author jingzz
	 * @param jarPath 生成的jar的位置
	 * @param conf 传入Configuration类
	 */
	public static void setLocalDebugConfiguration(String jarPath, Configuration conf) {
		
		conf.set("mapreduce.framework.name", "yarn");  
		/*
		 * 避免/tmp/hadoop-yarn/staging/JING/.staging/job_1481726235168_0009/job.splitmetainfo找不到的错误
		 * value为hdfs-site.xml的dfs.namenode.rpc-address.ns1属性指定的值。
		 */
		conf.set("fs.defaultFS", "hdfs://h2m1:8220");  
		
		//避免ExitCodeException exitCode=1: /bin/bash: line 0: fg: no job control
		conf.set("mapred.remote.os", "Linux");  
		conf.set("mapreduce.app-submission.cross-platform", "true"); 
		
		/*
		 * 避免Retrying connect to server: 0.0.0.0/0.0.0.0:8030
		 * 一般为默认设置，value为主yarn的resourcemanager节点的host:port
		 */
		conf.set("yarn.resourcemanager.scheduler.address", "h2m1:8030");  
		conf.set("yarn.resourcemanager.address", "192.168.2.201:8032");
		
		/*
		 * 避免java.lang.ClassNotFoundException: Class cn.followtry.hadoop.demo.v2.mr.WordCountMapV2 not found问题
		 * 日志提示：No job jar file set.  User classes may not be found. See JobConf(Class) or JobConf#setJar(String)
		 * 所以不管是在Eclipse中直接调试运行还是上传到Yarn集群，都需要先打成Jar包。
		 * "mapred.jar"不需要动，只要将"d:\\mapreduce.jar"处替换为打好包的jar的位置。
		 * "mapred.jar" 已经过期，请替换为"mapreduce.job.jar"
		 */
		//conf.set("mapred.jar", "d:\\mapreduce.jar");
		conf.set("mapreduce.job.jar", jarPath);
	}
	
}

目录webhdfs://h2m1:50070/user/root/output不存在
 INFO - 目录webhdfs://h2m1:50070/user/root/output不存在
  INFO - Connecting to ResourceManager at /192.168.2.201:8032
  WARN - Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this.
  INFO - Total input paths to process : 2
  INFO - number of splits:2
  INFO - Submitting tokens for job: job_1481717720063_0021
  INFO - Submitted application application_1481717720063_0021
  INFO - The url to track the job: http://h2m1:8088/proxy/application_1481717720063_0021/
  INFO - Running job: job_1481717720063_0021
  INFO - Job job_1481717720063_0021 running in uber mode : false
  INFO -  map 0% reduce 0%
  INFO -  map 50% reduce 0%
  INFO -  map 100% reduce 0%
  INFO -  map 100% reduce 100%
  INFO - Job job_1481717720063_0021 completed successfully