0050-mapreduce入门-word count

清疚 2023-07-14 10:56 98阅读 0赞

### 文章目录 ###

*   *   *  1. 说明
         *  2. 添加依赖
         *  3. 编写代码
         *   *  3.1 Mapper程序
             *  3.2 Reducer程序
             *  3.3 job类
         *  4. 打包
         *   *  5. 执行

### 1. 说明 ###

hello java
    hello js
    hello hadoop
    hello html
    hello hdfs
    hello mr
    hello hive

用mapper-reduce程序统计上述数据中各单词出现的次数  
![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dyb25neWFv_size_16_color_FFFFFF_t_70]注：  
map程序首先切分单词，其次遍历单词数据输出<单词，1>，map会缓存结果，处理完文本以后统一输出<单词，1，1……>；reduce程序接受map输出的结果做求和运算。

### 2. 添加依赖 ###

<dependencies>
            
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-common</artifactId>
                <version>3.2.0</version>
            </dependency>
    
            <dependency>
                <groupId>junit</groupId>
                <artifactId>junit</artifactId>
                <version>4.13</version>
                <scope>test</scope>
            </dependency>
    
            <dependency>
                <groupId>org.slf4j</groupId>
                <artifactId>slf4j-log4j12</artifactId>
                <version>1.7.2</version>
            </dependency>
    
            <dependency>
                <groupId>org.projectlombok</groupId>
                <artifactId>lombok</artifactId>
                <version>1.18.10</version>
                <scope>provided</scope>
            </dependency>
            
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-mapreduce-client-core</artifactId>
                <version>3.2.0</version>
            </dependency>
            
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-mapreduce-client-common</artifactId>
                <version>3.2.0</version>
            </dependency>
            
            <dependency>
                <groupId>org.apache.hadoop</groupId>
                <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
                <version>3.2.0</version>
            </dependency>
        </dependencies>

### 3. 编写代码 ###

#### 3.1 Mapper程序 ####

mapper程序，继承Mapper类，有四个泛型：1和2两个是mapperreduce框架输入的类型，分别是LongWritable和Text表示输入文本的偏移量和行内容；3和4个是Text, LongWritable属于用户自定义类型这边表示单词和计数值1。

import org.apache.commons.lang3.StringUtils;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Mapper;
    
    import java.io.IOException;
    
    /** * create by rongyao * at 2/5/20 * Mapper的输入是框架固定的行偏移量，行的内容 * 输出是单词，计数值 */
    public class WCMapper extends Mapper<LongWritable, Text, Text, LongWritable> { 
    
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException { 
            String[] words = StringUtils.split(value.toString(), " ");
            for (String word : words) { 
                context.write(new Text(word), new LongWritable(1));
            }
        }
    }

#### 3.2 Reducer程序 ####

reducer程序继承Reducer类吗，也是4个泛型，1和2两个是mapper的输出类型，也是reducer程序的输入类型，3和4两个是最终输出类型。

import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Reducer;
    
    import java.io.IOException;
    
    /** * create by rongyao * at 2/5/20 * 输入是mapper的输出单词和计数列表 * 输出是单词，统计数 */
    public class WCReducer extends Reducer<Text, LongWritable, Text, LongWritable> { 
        @Override
        protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException { 
            long count = 0;
            for (LongWritable value : values) { 
                count += value.get();
            }
            context.write(key, new LongWritable(count));
        }
    }

#### 3.3 job类 ####

一个执行jar包中有很多mapper和reducer程序，所以需要一个job来指定当前任务是哪个mapper程序和reducer程序。

import org.apache.hadoop.conf.Configuration;
    import org.apache.hadoop.fs.Path;
    import org.apache.hadoop.io.LongWritable;
    import org.apache.hadoop.io.Text;
    import org.apache.hadoop.mapreduce.Job;
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
    
    /** * create by rongyao * at 2/5/20 */
    public class WCRunner { 
        public static void main(String[] args) throws Exception { 
            Configuration configuration = new Configuration();
            Job job = Job.getInstance(configuration);
            // 设置整个job所用的类所在位置
            job.setJarByClass(WCRunner.class);
    
            // 设置mapper和reducer类
            job.setMapperClass(WCMapper.class);
            job.setReducerClass(WCReducer.class);
    
            // 设置mapper输出类型
            job.setMapOutputKeyClass(Text.class);
            job.setMapOutputValueClass(LongWritable.class);
    
            // 设置reducer输出类型
            job.setOutputKeyClass(Text.class);
            job.setMapOutputValueClass(LongWritable.class);
    
            // 设置输入数据路径
            FileInputFormat.setInputPaths(job, new Path("/wc/input/"));
            // 设置处理结果输出路径
            FileOutputFormat.setOutputPath(job, new Path("/wc/output/"));
    
            // 提交job true 打印运行进度
            job.waitForCompletion(true);
        }
    }

### 4. 打包 ###

使用maven的打包命令 mvn clean package

#### 5. 执行 ####

将待统计数据上传到/wc/input/目录下，执行hadoop jar wc.jar com.honor.wccount.WCRunner， 结果会在/wc/output/下自动生成  
结果如下  
![在这里插入图片描述][20200314165550522.png]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dyb25neWFv_size_16_color_FFFFFF_t_70]: /images/20230529/ea8eec585700490683b3434a1c1175a7.png
[20200314165550522.png]: /images/20230529/c2ae06cfd3d34c54a6233d29d1387063.png