统计手机流量

布满荆棘的人生 2022-02-26 00:44 388阅读 0赞

### 统计手机流量 ###

*  原始数据
 *  需求
 *  编码实现
 *   *  Mapper
     *  Reduce
     *  TrafficCount

# 原始数据 #

1363157985066   13726230503     00-FD-07-A4-72-B8:CMCC  120.196.100.82  i02.c.aliimg.com                24      27      2481    24681   200
    1363157995052   13826544101     5C-0E-8B-C7-F1-E0:CMCC  120.197.40.4                    4       0       264     0       200
    1363157991076   13926435656     20-10-7A-28-CC-0A:CMCC  120.196.100.99                  2       4       132     1512    200
    1363154400022   13926251106     5C-0E-8B-8B-B1-50:CMCC  120.197.40.4                    4       0       240     0       200
    1363157993044   18211575961     94-71-AC-CD-E6-18:CMCC-EASY     120.196.100.99  iface.qiyi.com  视频网站        15      12      1527    2106    200
    1363157995074   84138413        5C-0E-8B-8C-E8-20:7DaysInn      120.197.40.4    122.72.52.12            20      16      4116    1432    200
    1363157993055   13560439658     C4-17-FE-BA-DE-D9:CMCC  120.196.100.99                  18      15      1116    954     200
    1363157995033   15920133257     5C-0E-8B-C7-BA-20:CMCC  120.197.40.4    sug.so.360.cn   信息安全        20      20      3156    2936    200
    1363157983019   13719199419     68-A1-B7-03-07-B1:CMCC-EASY     120.196.100.82                  4       0       240     0       200
    1363157984041   13660577991     5C-0E-8B-92-5C-20:CMCC-EASY     120.197.40.4    s19.cnzz.com    站点统计        24      9       6960    690     200
    1363157973098   15013685858     5C-0E-8B-C7-F7-90:CMCC  120.197.40.4    rank.ie.sogou.com       搜索引擎        28      27      3659    3538    200
    1363157986029   15989002119     E8-99-C4-4E-93-E0:CMCC-EASY     120.196.100.99  www.umeng.com   站点统计        3       3       1938    180     200
    1363157992093   13560439658     C4-17-FE-BA-DE-D9:CMCC  120.196.100.99                  15      9       918     4938    200
    1363157986041   13480253104     5C-0E-8B-C7-FC-80:CMCC-EASY     120.197.40.4                    3       3       180     180     200
    1363157984040   13602846565     5C-0E-8B-8B-B6-00:CMCC  120.197.40.4    2052.flash2-http.qq.com 综合门户        15      12      1938    2910    200
    1363157995093   13922314466     00-FD-07-A2-EC-BA:CMCC  120.196.100.82  img.qfc.cn              12      12      3008    3720    200
    1363157982040   13502468823     5C-0A-5B-6A-0B-D4:CMCC-EASY     120.196.100.99  y0.ifengimg.com 综合门户        57      102     7335    110349  200
    1363157986072   18320173382     84-25-DB-4F-10-1A:CMCC-EASY     120.196.100.99  input.shouji.sogou.com  搜索引擎        21      18      9531    2412    200
    1363157990043   13925057413     00-1F-64-E1-E6-9A:CMCC  120.196.100.55  t3.baidu.com    搜索引擎        69      63      11058   48243   200
    1363157988072   13760778710     00-FD-07-A4-7B-08:CMCC  120.196.100.82                  2       2       120     120     200
    1363157985066   13726238888     00-FD-07-A4-72-B8:CMCC  120.196.100.82  i02.c.aliimg.com                24      27      2481    24681   200
    1363157993055   13560436666     C4-17-FE-BA-DE-D9:CMCC  120.196.100.99                  18      15      1116    954     200

# 需求 #

> 统计出每个手机号的流量

# 编码实现 #

## Mapper ##

public class TrafficCountMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
            @Override
            protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
                    String line = value.toString();
                    String[] fileds = line.split("\t");
                    if(fileds.length!=11) return;
    
                    context.write(new Text(fileds[1]), new IntWritable(Integer.parseInt(fileds[9])));
    
            }
    }

## Reduce ##

public class TrafficCountReducer  extends Reducer< Text, IntWritable,  Text, IntWritable>{
            @Override
            protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
                    int traffic = 0;
                    for (IntWritable value : values) {
                            traffic += value.get();
                    }
                    context.write(key, new IntWritable(traffic));
            }
    }

## TrafficCount ##

public class TrafficCount {
            public static void main(String[] args) throws Exception{
                    Configuration config = new Configuration();
                    Job job = Job.getInstance(config);
    
                    job.setJarByClass(TrafficCount.class);
    
                    job.setMapperClass(TrafficCountMapper.class);
                    job.setReducerClass(TrafficCountReducer.class);
    
                    job.setMapOutputKeyClass(Text.class);
                    job.setMapOutputValueClass(IntWritable.class);
    
                    job.setOutputKeyClass(Text.class);
                    job.setOutputValueClass(IntWritable.class);
    
                    /*FileInputFormat.setInputPaths(job, new Path("hdfs://192.168.56.160:9000/traffic/input/traffic.dat"));
                    FileOutputFormat.setOutputPath(job, new Path("hdfs://192.168.56.160:9000/traffic/output"));*/
                    FileInputFormat.setInputPaths(job, new Path("/home/hasee/bigdata/traffic/input/traffic.dat"));
                    FileOutputFormat.setOutputPath(job, new Path("/home/hasee/bigdata/traffic/output"));
                    boolean res = job.waitForCompletion(true);
                    System.exit(res?0:1);
            }
    }