美文网首页大数据BigData
HBase处理通话记录

HBase处理通话记录

作者: geekAppke | 来源:发表于2018-12-05 22:42 被阅读16次
phoneNum 主机号
dnum 对方手机号码
length 时长
type 类型:主叫 / 被叫
date 时间

时间戳 = 手机号 + (MAX - 时间)
字典顺序,时间越近,放在最上面最好!

HBase客户端Java-API

生产通话记录

    SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");

    // 有10个用户,每个用户随机产生100条记录(批量插入)
    @Test
    public void insertDB2() throws Exception {
        List<Put> puts = new ArrayList<Put>();
        for (int i = 0; i < 10; i++) {
            String phoneNum = getPhoneNum("186");
            for (int j = 0; j < 100; j++) {
                String dnum = getPhoneNum("158");
                String length = r.nextInt(99) + "";
                String type = r.nextInt(2) + "";
                String dateStr = getDate("2018");
                String rowkey = phoneNum + "_" + (Long.MAX_VALUE - sdf.parse(dateStr).getTime());

                Put put = new Put(rowkey.getBytes());
                put.add("cf".getBytes(), "dnum".getBytes(), dnum.getBytes());
                put.add("cf".getBytes(), "length".getBytes(), length.getBytes());
                put.add("cf".getBytes(), "type".getBytes(), type.getBytes());
                put.add("cf".getBytes(), "date".getBytes(), dateStr.getBytes());
                puts.add(put);
            }
        }
        htable.put(puts);
    }

    Random r = new Random();
    // 生成随机的手机号码
    private String getPhoneNum(String string) {
        return string + String.format("%08d", r.nextInt(99999999));
    }
    private String getDate(String year) {
        return year + String.format("%02d%02d%02d%02d%02d", new Object[] { r.nextInt(12) + 1, r.nextInt(31) + 1, r.nextInt(24), r.nextInt(60), r.nextInt(60) });
    }

    @Test
    public void getDB2() throws Exception{
        Get get = new Get("18686966381_9223370509257224807".getBytes());
        Result result = htable.get(get);
        Cell cell = result.getColumnLatestCell("cf".getBytes(), "day".getBytes());
        Phone.dayPhoneDetail dayPhone = Phone.dayPhoneDetail.parseFrom(CellUtil.cloneValue(cell));
        for (PhoneDetail pd : dayPhone.getDayPhoneDetailList()) {
            System.out.println(pd);
        }
    }

统计二月份到三月份的通话记录

    @Test
    public void scan() throws Exception {
        String phoneNum = "18676604687";
        String startRow = phoneNum + "_" + (Long.MAX_VALUE - sdf.parse("20180301000000").getTime());
        String stopRow = phoneNum + "_" + (Long.MAX_VALUE - sdf.parse("20180201000000").getTime());
        Scan scan = new Scan();
        scan.setStartRow(startRow.getBytes());
        scan.setStopRow(stopRow.getBytes());
        ResultScanner rss = htable.getScanner(scan);
        for (Result rs : rss) {
            System.out.print(new String(CellUtil.cloneValue(rs.getColumnLatestCell("cf".getBytes(), "dnum".getBytes()))));
            System.out.print("-"+ new String(CellUtil.cloneValue(rs.getColumnLatestCell("cf".getBytes(), "length".getBytes()))));
            System.out.print("-" + new String(CellUtil.cloneValue(rs.getColumnLatestCell("cf".getBytes(), "type".getBytes()))));
            System.out.println("-" + new String(CellUtil.cloneValue(rs.getColumnLatestCell("cf".getBytes(), "date".getBytes()))));
        }
    }

查询某个手机号主叫为1 的所有记录

条件查询,过滤器

    @Test
    public void scan2() throws Exception {
        FilterList list = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        PrefixFilter filter1 = new PrefixFilter("18676604687".getBytes());
        SingleColumnValueFilter filter2 = new SingleColumnValueFilter("cf".getBytes(), "type".getBytes(), CompareOp.EQUAL, "1".getBytes());
        list.addFilter(filter1);
        list.addFilter(filter2);
        Scan scan = new Scan();
        scan.setFilter(list);
        ResultScanner rss = htable.getScanner(scan);
        for (Result rs : rss) {
            System.out.print(new String(CellUtil.cloneValue(rs.getColumnLatestCell("cf".getBytes(), "dnum".getBytes()))));
            System.out.print("-"+ new String(CellUtil.cloneValue(rs.getColumnLatestCell("cf".getBytes(), "length".getBytes()))));
            System.out.print("-" + new String(CellUtil.cloneValue(rs.getColumnLatestCell("cf".getBytes(), "type".getBytes()))));
            System.out.println("-" + new String(CellUtil.cloneValue(rs.getColumnLatestCell("cf".getBytes(), "date".getBytes()))));
        }
    }
待优化?减少rowKey


模拟数据2

    @Test
    public void insertDB3() throws Exception {
        List<Put> puts = new ArrayList<Put>();
        for (int i = 0; i < 10; i++) {
            String phoneNum = getPhoneNum("186");
            for (int j = 0; j < 100; j++) {
                String dnum = getPhoneNum("158");
                String length = r.nextInt(99) + "";
                String type = r.nextInt(2) + "";
                String dateStr = getDate("2018");
                String rowkey = phoneNum + "_" + (Long.MAX_VALUE - sdf.parse(dateStr).getTime());
                Phone2.PhoneDetail.Builder phoneDetail = Phone2.PhoneDetail.newBuilder();
                phoneDetail.setDate(dateStr);
                phoneDetail.setDnum(dnum);
                phoneDetail.setLength(length);
                phoneDetail.setType(type);
                Put put = new Put(rowkey.getBytes());
                put.add("cf".getBytes(), "phoneDetail".getBytes(), phoneDetail.build().toByteArray());
                puts.add(put);
            }
        }
        htable.put(puts);
    }

    /**
     * 有十个用户,每个用户每天产生100条记录,将100条记录放到一个集合进行存储
     * 
     * @throws Exception
     */
    @Test
    public void insertDB4() throws Exception {
        List<Put> puts = new ArrayList<Put>();
        for (int i = 0; i < 10000; i++) {
            String phoneNum = getPhoneNum("186");
            String rowkey = phoneNum + "_" + (Long.MAX_VALUE - sdf.parse(getDate2("20180529")).getTime());
            Phone.dayPhoneDetail.Builder dayPhone = Phone.dayPhoneDetail.newBuilder();
            for (int j = 0; j < 100; j++) {
                String dnum = getPhoneNum("158");
                String length = r.nextInt(99) + "";
                String type = r.nextInt(2) + "";
                String dateStr = getDate("2018");
                Phone.PhoneDetail.Builder phoneDetail = Phone.PhoneDetail.newBuilder();
                phoneDetail.setDate(dateStr);
                phoneDetail.setDnum(dnum);
                phoneDetail.setLength(length);
                phoneDetail.setType(type);
                dayPhone.addDayPhoneDetail(phoneDetail);
            }
            Put put = new Put(rowkey.getBytes());
            put.add("cf".getBytes(), "day".getBytes(), dayPhone.build().toByteArray());
            puts.add(put);
        }
        htable.put(puts);
    }
    
    private String getDate2(String yearMonthDay) {
        return yearMonthDay
                + String.format("%02d%02d%02d", new Object[] { r.nextInt(24), r.nextInt(60), r.nextInt(60) });
    }
通话记录

参考资料

Hadoop集群搭建高可用HBase
HBase的Shell - 简书
HBase客户端Java-API

相关文章

网友评论

    本文标题:HBase处理通话记录

    本文链接:https://www.haomeiwen.com/subject/hzugcqtx.html