Java8中lambad表达式去重

resultList.stream().distinct().collect(Collectors.toList());

List转map

Function.identity()是代表本身的意思

List<TaskDto> csvTaskDtos = new ArrayList<>();
Map<String, TaskDto> csvTaskDtoMap =
                csvTaskDtos.stream().collect(Collectors.toMap(TaskDto::getThemeJstId, Function.identity()));
两个list取并集 各自单独的独有属性

   Set<String> systemTaskDtoMapKey = systemTaskDtoMap.keySet();
   Set<String> csvTaskDtoMapKey = csvTaskDtoMap.keySet();
   Set<String> includeBothKey =
                systemTaskDtoMapKey.stream().filter(k -> csvTaskDtoMapKey.contains(k)).collect(Collectors.toSet());
   Set<String> includeSystemKey =
                systemTaskDtoMapKey.stream().filter(k -> !csvTaskDtoMapKey.contains(k)).collect(Collectors.toSet());
   Set<String> includeCsvKey =
                csvTaskDtoMapKey.stream().filter(k -> !systemTaskDtoMapKey.contains(k)).collect(Collectors.toSet());

示例

package com.lambda;

import com.alibaba.fastjson.JSON;

import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.TreeSet;
import java.util.stream.Collectors;

/**
 * @author Calebit
 * @date 2020-06-06 21:43
 */
public class LambadMainTest {

    public static void main(String[] args) {
        Rule rule1 = new Rule(1001, "sign", "day1", 1);
        Rule rule2 = new Rule(1002, "sign", "day2", 2);
        Rule rule3 = new Rule(1003, "sign", "day3", 3);
        Rule rule4 = new Rule(1004, "sign", "day4", 4);

        List<Rule> newList = new ArrayList<>();
        newList.add(rule1);
        newList.add(rule2);
        newList.add(rule3);
        newList.add(rule4);

        Rule rule21 = new Rule(2001, "sign", "day1", 21);
        Rule rule23 = new Rule(2003, "sign", "day3", 23);
        Rule rule26 = new Rule(2006, "sign", "day6", 26);
        Rule rule27 = new Rule(2007, "sign", "day7", 27);

        List<Rule> oldList = new ArrayList<>();
        oldList.add(rule21);
        oldList.add(rule23);
        oldList.add(rule26);
        oldList.add(rule27);


        //需要更新的
        List<Rule> collect = newList.stream().filter(n -> oldList.stream().filter(o ->
                  n.getPointType().equals(o.getPointType()) && n.getRuleType().equals(o.getRuleType())
        ).findFirst().isPresent()).collect(Collectors.toList());
        System.out.println("需要更新的 = " + JSON.toJSONString(collect));

        //需要删除的
        List<Rule> collect2 = oldList.stream().filter( o ->
                    !newList.stream().filter(n -> n.getPointType().equals(o.getPointType())).findAny().isPresent()
        ).collect(Collectors.toList());
        System.out.println("需要删除的 = " + JSON.toJSONString(collect2));


        //需要新增加的
        List<Rule> collect3 = newList.stream().filter( o ->
                !oldList.stream().filter(n -> n.getPointType().equals(o.getPointType())).findAny().isPresent()
        ).collect(Collectors.toList());

        System.out.println("需要新增加的 = " + JSON.toJSONString(collect3));


        //单个List去重,注意这里规则,哪个先添加进list,那么结果就是以 先添加进list 的为准
//        List<Rule> mergeList = new ArrayList<>(newList);
//        mergeList.addAll(oldList);
        
        List<Rule> mergeList = new ArrayList<>(oldList);
        mergeList.addAll(newList);
        List<Rule> distinctList = mergeList.stream()
          .collect(Collectors.collectingAndThen(
                    Collectors.toCollection(() -> new TreeSet<>(Comparator.comparing(
                                   o -> o.getRuleType() + ";" + o.getPointType()))), ArrayList::new));
        System.out.println("单个List去重 = " + JSON.toJSONString(distinctList));

    }

}

class Rule {
    private Integer id;
    private String ruleType;
    private String pointType;
    private Integer pointVal;
    public Rule() {
    }

    public Rule(Integer id, String ruleType, String pointType, Integer pointVal) {
        this.id = id;
        this.ruleType = ruleType;
        this.pointType = pointType;
        this.pointVal = pointVal;
    }

    public Integer getId() {
        return id;
    }

    public void setId(Integer id) {
        this.id = id;
    }

    public String getRuleType() {
        return ruleType;
    }

    public void setRuleType(String ruleType) {
        this.ruleType = ruleType;
    }

    public String getPointType() {
        return pointType;
    }

    public void setPointType(String pointType) {
        this.pointType = pointType;
    }

    public Integer getPointVal() {
        return pointVal;
    }

    public void setPointVal(Integer pointVal) {
        this.pointVal = pointVal;
    }
}

实现java8 list按照元素的某个字段去重
原文：https://www.jb51.net/article/163422.htm?tdsourcetag=s_pcqq_aiomsg

 @Data
@AllArgsConstructor
@NoArgsConstructor
public class Student {
private Integer age;
private String name;
}

List<Student> studentList = Lists.newArrayList();
studentList.add(new Student(28, "river"));
studentList.add(new Student(12, "lucy"));
studentList.add(new Student(33, "frank"));
studentList.add(new Student(33, "lucy"));

多个字段去重

List<Student> studentDistinctList = studentList.stream()
.collect(Collectors.collectingAndThen
(Collectors.toCollection(() ->
new TreeSet<>(Comparator.comparing(t -> t.getName() + ";" + t.getAge() ))),
ArrayList::new));

System.out.println(new Gson().toJson(studentDistinctList));

扩展distinct 方法去重

List<Student> studentDistinct2List = studentList.stream().filter(StreamUtil.distinctByKey(t->t.getName()))
.collect(Collectors.toList());
System.out.println(new Gson().toJson(studentDistinct2List));

工具类StreamUtil.java

public class StreamUtil {
/**
* https://stackoverflow.com/questions/23699371/java-8-distinct-by-property
* @param keyExtractor
* @param <T>
* @return
*/
public static <T> Predicate<T> distinctByKey(Function<? super T, ?> keyExtractor) {
  Set<Object> seen = ConcurrentHashMap.newKeySet();
  return t -> seen.add(keyExtractor.apply(t));
  }
}

获取重复的数据

@Test
public void test10(){
//根据device_code去重，取出重复值
    List<Employee> emps =new ArrayList<>();
            List<String> dupList = emps.stream().collect(Collectors.groupingBy(Employee::getName, Collectors.counting()))
            .entrySet().stream().filter(e -> e.getValue() > 1)
            .map(Map.Entry::getKey).collect(Collectors.toList());
    System.out.println(dupList);
}
@Test
public void test11(){
    //字符串取出重复值
    List<String> list = new ArrayList<>();
    List<String> repeatList = list.stream().collect(Collectors.groupingBy(e -> e, Collectors.counting()))
            .entrySet().stream().filter(e -> e.getValue() > 1)
            .map(Map.Entry::getKey).collect(Collectors.toList());
    System.out.println(repeatList);
}