Hadoop3.2.0 idea 远程提交 mapreduce job 任务 (手写wordcount)

悠悠 2021-11-05 19:54 359阅读 0赞

创建maven项目导入jar包

<dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-common</artifactId>
                <version>3.2.0</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-hdfs</artifactId>
                <version>3.2.0</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-client</artifactId>
                <version>3.2.0</version>
            </dependency>
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-mapreduce-client-core</artifactId>
                <version>3.2.0</version>
            </dependency>
            <dependency>
                <groupId>commons-logging</groupId>
                <artifactId>commons-logging</artifactId>
                <version>1.2</version>
            </dependency>

将服务器hadoop集群配置文件拷贝到项目resources目录下

这里要注意,映射名需要改成服务器IP地址(或者在本地编写IP映射关系)

项目结构如下

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MDQ2MTI4MQ_size_16_color_FFFFFF_t_70][]

在本地创建环境变量 HADOOP\_USER\_NAME 值为集群设置的执行用户

![20190809101418707.png][]

在Path中添加 HADOOP\_USER\_NAME 变量

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MDQ2MTI4MQ_size_16_color_FFFFFF_t_70 1][]

MyWordCount.java

import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.FileSystem;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    
    public class MyWordCount {
    
        public static void main(String[] args){
            Configuration conf = new Configuration(true);
            conf.addResource("core-site.xml");
            conf.addResource("hdfs-site.xml");
            conf.addResource("mapred-site.xml");
            conf.addResource("yarn-site.xml");
            // 也可以在这里设置刚刚编译好的jar
            conf.set("mapred.jar", "target/MRdemo-1.0-SNAPSHOT.jar");
            // Windows开发者需要设置跨平台
            conf.set("mapreduce.app-submission.cross-platform", "true");
            try {
                // 创建一个job
                Job job = Job.getInstance(conf);
                job.setJarByClass(MyWordCount.class);
                job.setJobName("wordcount");
    
                // 数据文件输入路径
                Path input = new Path("/user/root/test.txt");
                FileInputFormat.addInputPath(job,input);
                // 结果输出路径
                Path output = new Path("/root/output");
                FileSystem fileSystem = output.getFileSystem(conf);
                if (fileSystem.exists(output)){
                    fileSystem.delete(output,true);
                }
                FileOutputFormat.setOutputPath(job,output);
    
                job.setMapperClass(MyMapper.class);
                job.setMapOutputKeyClass(Text.class);
                job.setMapOutputValueClass(IntWritable.class);
    
                job.setReducerClass(MyReducer.class);
    
                job.waitForCompletion(true);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

MyMapper.java

import	java.util.StringTokenizer;
    
    import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Mapper;
    
    import java.io.IOException;
    
    public class MyMapper extends Mapper<Object, Text, Text, IntWritable> {
    
        private static final IntWritable ONE = new IntWritable(1);
        private Text word = new Text();
    
        @Override
        public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
            StringTokenizer itr = new StringTokenizer(value.toString());
            while(itr.hasMoreTokens()){
                word.set(itr.nextToken());
                context.write(word,ONE);
            }
        }
    }

MyReducer.java

import org.apache.hadoop.io.IntWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Reducer;
    
    import java.io.IOException;
    
    
    public class MyReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
        /**
         * 相同的key为一组 ... 调用一次reduce方法,在方法内迭代这一组数据,进行:sum count max min ...
         */
        private IntWritable result = new IntWritable();
    
        @Override
        public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
            int sum = 0;
            for (IntWritable value : values){
                sum += value.get();
            }
            result.set(sum);
            context.write(key,result);
        }
    }

使用idea maven 工具将项目打包

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MDQ2MTI4MQ_size_16_color_FFFFFF_t_70 2][]

执行 MyWordCount main方法

在 resourcemanager web管理端 查看执行进度

我的 resourcemanager 地址为 192.168.0.200:8088

![20190809104441745.png][]

在 hadoop 服务器查看是否生成结果文件

hdfs dfs -ls -R /root/output

![20190809104907847.png][]

在服务器使用 命令查看结果文件内容

hdfs dfs -cat /root/output/part-r-00000

![2019080910502284.png][]

结果正确,执行成功

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MDQ2MTI4MQ_size_16_color_FFFFFF_t_70]: /images/20211104/933bee1213a644238e460cd782cffa73.png
[20190809101418707.png]: /images/20211104/b38478f5b574401fb8c512cb35be6b1b.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MDQ2MTI4MQ_size_16_color_FFFFFF_t_70 1]: /images/20211104/cf80ba11c7b54e8aa2715a255b45be36.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MDQ2MTI4MQ_size_16_color_FFFFFF_t_70 2]: /images/20211104/77dbca49d1e94649a2404f4bab9bf9d1.png
[20190809104441745.png]: /images/20211104/1139e00b68db47a88f4cb7e49827c521.png
[20190809104907847.png]: /images/20211104/80aae759f958420090b7a1661c500dc0.png
[2019080910502284.png]: /images/20211104/f288246baa374b558a8c03fe84d7c2ae.png