ES(elasticsearch) springboot+es

作者: 凡哥爱丽姐 | 来源:发表于2022-03-26 21:27 被阅读0次

创建一个springboot项目

整体架构

1、导入对应的依赖

    <dependencies>
        <!--elasticsearch需要的包-->
<!--        <dependency>-->
<!--            <groupId>org.springframework.boot</groupId>-->
<!--            <artifactId>spring-boot-starter-data-elasticsearch</artifactId>-->
<!--        </dependency>-->

        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-high-level-client</artifactId>
            <version>6.4.2</version>
        </dependency>
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-client</artifactId>
            <version>6.4.2</version>
        </dependency>
        <dependency>
            <groupId>org.elasticsearch</groupId>
            <artifactId>elasticsearch</artifactId>
            <!--版本保持一致，不然缺类-->
            <version>6.4.2</version>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-devtools</artifactId>
            <scope>runtime</scope>
            <optional>true</optional>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <optional>true</optional>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.70</version>
         </dependency>
    </dependencies>

2、实体类User

package com.example.demo.pojo;

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

@Data
@NoArgsConstructor
@AllArgsConstructor
public class User {
    private String name;
    private Integer age;
}

3、ElasticSearchClientConfig来配置一个相应的类

package com.example.demo.config;

import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

@Configuration
public class ElasticSearchClientConfig {

    @Bean
    public RestHighLevelClient restHighLevelClient(){
        RestHighLevelClient client=new RestHighLevelClient(
                RestClient.builder(new HttpHost("127.0.0.1",9200,"http")));
        return client;
    }
}

4、写一个测试类去测试

package com.example.demo;

import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;

@RunWith(SpringRunner.class)
@SpringBootTest
public class DemoApplicationTests {
    @Autowired
    @Qualifier("restHighLevelClient")
    private RestHighLevelClient client;



}

5、创建索引

详情见：https://www.elastic.co/guide/en/elasticsearch/client/java-rest/7.9/java-rest-high-create-index.html

//测试创建索引
    @Test
    public void contextLoads() throws IOException {
        //创建 索引"weifan" 请求
        CreateIndexRequest request = new CreateIndexRequest("weifan");
        //客户端执行请求，并获得相应
        CreateIndexResponse createIndexResponse
                = client.indices().create(request, RequestOptions.DEFAULT);

        System.out.println(createIndexResponse);

    }

6、获取索引

详情见：https://www.elastic.co/guide/en/elasticsearch/client/java-rest/7.9/java-rest-high-indices-exists.html

    //测试获取索引
    @Test
    public void testExistIndex() throws IOException{
//      GetRequest request=new GetRequest("weifan","","");
//      boolean exists = client.exists(request, RequestOptions.DEFAULT);
//      System.out.println(exists);
        //1、创建请求对象
        GetIndexRequest request=new GetIndexRequest();
        request.indices("weifan");
        //判断
        boolean exists
                = client.indices().exists(request, RequestOptions.DEFAULT);
        System.out.println(exists);

    }

7、删除索引

详情见：https://www.elastic.co/guide/en/elasticsearch/client/java-rest/7.9/java-rest-high-delete-index.html

    //测试删除索引
    @Test
    public void testDeleteIndex() throws IOException{
        DeleteIndexRequest request=new DeleteIndexRequest("weifan");
        DeleteIndexResponse delete = client.indices().delete(request,RequestOptions.DEFAULT);
        System.out.println(delete.isAcknowledged());
    }

8、添加文档

详情见：https://www.elastic.co/guide/en/elasticsearch/client/java-rest/7.9/java-rest-high-document-index.html

@Test
    public void testAddDocument() throws IOException {
        User user = new User("狂神说", 3);
        //创建索引请求
        IndexRequest request= new IndexRequest("kuangshen");
        //IndexRequest request1=new IndexRequest("kuangshen","User","1");
        //kibana 会用 put/kuangshen/User/1  添加数据
        //           {
        //              "name":"狂神说",
        //              "age":23
        //            }
        request.type("User");
        request.id("1");
        //将我们的数据放入请求json中(指定添加的数据)
        request.source(JSON.toJSONString(user), XContentType.JSON);

        //客户端发送请求，获取响应的结果
        IndexResponse indexResponse = client.index(request, RequestOptions.DEFAULT);

        System.out.println(indexResponse.getIndex());//kuangshen
        System.out.println(indexResponse.toString());//IndexResponse[index=kuangshen,type=User,id=1,version=1,result=created,seqNo=0,primaryTerm=1,shards={"total":2,"successful":1,"failed":0}]
        System.out.println(indexResponse.status());//对应我们命令返回状态是CREATED
    }

9、获取文档，判断是否存在

@Test
    public void testIsExists() throws IOException {
        GetRequest getRequest = new GetRequest("kuangshen","User", "1");
        boolean exists = client.exists(getRequest, RequestOptions.DEFAULT);
        System.out.println(exists);
    }

10、获取文档信息

@Test
    public void testGetDocument() throws IOException{
        GetRequest getRequest = new GetRequest("kuangshen","User", "1");
        GetResponse getResponse = client.get(getRequest, RequestOptions.DEFAULT);
        //打印文档的内容
        System.out.println(getResponse.getSourceAsString());//{"age":3,"name":"狂神说"}
        System.out.println(getResponse);//{"_index":"kuangshen","_type":"User","_id":"1","_version":1,"found":true,"_source":{"age":3,"name":"狂神说"}}
    }

11、更新文档信息

//更新文档信息
    @Test
    public void testUpdateRequest() throws IOException {
        UpdateRequest updateRequest=new UpdateRequest("kuangshen","User","1");
        updateRequest.timeout("1s");

        User user=new User("狂神说Java",18);
        updateRequest.doc(JSON.toJSONString(user),XContentType.JSON);

        UpdateResponse updateResponse = client.update(updateRequest, RequestOptions.DEFAULT);
        System.out.println(updateResponse.status());//OK
        System.out.println(updateResponse.toString());//UpdateResponse[index=kuangshen,type=User,id=1,version=2,seqNo=-2,primaryTerm=0,result=noop,shards=ShardInfo{total=0, successful=0, failures=[]}]
        System.out.println(updateResponse);
    }

12、删除文档记录

@Test
    public void testDeleteRequest() throws IOException {

        DeleteRequest request = new DeleteRequest("kuangshen","User", "1");
        request.timeout("1s");

        DeleteResponse deleteResponse = client.delete(request, RequestOptions.DEFAULT);
        System.out.println(deleteResponse.status());
    }

13、批量插入数据

@Test
    public void testBulkRequest() throws IOException {
        BulkRequest bulkRequest=new BulkRequest();
        bulkRequest.timeout("10s");

        List<User> userList=new ArrayList<>();
        userList.add(new User("weifan1",3));
        userList.add(new User("weifan2",3));
        userList.add(new User("weifan3",3));
        userList.add(new User("weifan4",3));

        //批处理
        for(int i=0;i<userList.size();i++){
            bulkRequest.add(
                            new IndexRequest("kuangshen")
                                    .type("User")
                                    .id(""+(i+1))
                                    .source(JSON.toJSONString(userList.get(i)),XContentType.JSON));
        }

        BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
        System.out.println(bulkResponse.hasFailures());
    }

14、查询

详情见：https://www.elastic.co/guide/en/elasticsearch/client/java-rest/7.9/java-rest-high-search.html

例14.1

    //查询
    @Test
    public void testSearch() throws IOException {
        SearchRequest searchRequest = new SearchRequest("kuangshen");
        //构建搜索条件
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        //sourceBuilder.sort() 排序查询


        //查询条件，我们可以使用QueryBuilders工具来实现
        //QueryBuilders.termQuery() 精确
        //QueryBuilders.matchAllQuery() 匹配所有
        //QueryBuilders.fuzzyQuery() 模糊查询
        //QueryBuilders.rangeQuery() 范围查询
        TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("name", "weifan1");
        sourceBuilder.query(termQueryBuilder);
        sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
        
        searchRequest.source(sourceBuilder);

        SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
        
        System.out.println(JSON.toJSONString(searchResponse.getHits()));
        //结果：{"fragment":true,"hits":[{"fields":{},"fragment":false,"highlightFields":{},"id":"1","matchedQueries":[],"score":0.2876821,"sortValues":[],"sourceAsMap":{"name":"weifan1","age":3},"sourceAsString":"{\"age\":3,\"name\":\"weifan1\"}","sourceRef":{"childResources":[],"fragment":true},"type":"User","version":-1}],"maxScore":0.2876821,"totalHits":1}
        System.out.println("=====================");
        for (SearchHit documentFields : searchResponse.getHits().getHits()) {
            System.out.println(documentFields.getSourceAsString());//{"age":3,"name":"weifan1"}
            //System.out.println(documentFields.getSourceAsMap());
        }

    }

例14.2

/**
     * ##搜索address中包含mill的所有年龄分布和平均年龄
     * GET bank/_search
     * {
     *   "query":{
     *    "match": {
     *      "address":"mill"
     *    }
     *   },
     *   "aggs": {
     *     "ageAgg": {
     *       "terms": {
     *         "field": "age",
     *         "size": 10
     *       }
     *     },
     *     "ageAvg":{
     *       "avg": {
     *         "field": "age"
     *       }
     *     }
     *   }
     * }
     * */

    @Test
    public void searchData() throws IOException {
        SearchRequest searchRequest = new SearchRequest("bank");

        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.query(QueryBuilders.termQuery("address", "mill"));

        TermsAggregationBuilder aggregation = AggregationBuilders.terms("ageAgg")//Aggregations聚合
                .field("age").size(10);
        aggregation.subAggregation(AggregationBuilders.avg("ageAvg")
                .field("age"));
        searchSourceBuilder.aggregation(aggregation);

        searchRequest.source(searchSourceBuilder);
        System.out.println(searchRequest);

        SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
        System.out.println(searchResponse);
    }

15、分页查询（from-size 适用于浅分页；分页越深，性能越差）

from-size浅分页适合数据量不大的情况（官网推荐是数据少于10000条）
详细请见官网：https://www.elastic.co/guide/cn/elasticsearch/guide/2.x/_query_phase.html
从下图可知es中有7条数据

es中有7条数据

     //分页查询
    @Test
    public void testSearchByPage1() throws IOException {
        Integer currentPage=1;
        Integer pageSize=3;

        SearchRequest searchRequest=new SearchRequest();
        searchRequest.indices("kuangshen");
        SearchSourceBuilder searchSourceBuilder=new SearchSourceBuilder();
        /**
         * 浅分页
         * GET /_search{
         * "from":0
         * "size":3
         *}
         */
        searchSourceBuilder.from(currentPage-1);
        searchSourceBuilder.size(pageSize);
        searchRequest.source(searchSourceBuilder);

        SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT);

        System.out.println(JSON.toJSONString(response.getHits()));
        System.out.println("------------------------------------------------------");
        SearchHit[] hits = response.getHits().getHits();
        for (SearchHit hit:hits) {
            System.out.println(hit.getSourceAsString());
        }
    }

    //浅分页多次查询（from-size）
    @Test
    public void testSearchByPage2() throws IOException {
        Integer currentPage=1;
        Integer pageSize=3;

        SearchRequest searchRequest=new SearchRequest();
        searchRequest.indices("kuangshen");
        SearchSourceBuilder searchSourceBuilder=new SearchSourceBuilder();
        searchSourceBuilder.from(currentPage-1);
        searchSourceBuilder.size(pageSize);

        Boolean hasMore=true;
        while (hasMore){
            searchRequest.source(searchSourceBuilder);
            SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT);
            //System.out.println(JSON.toJSONString(response.getHits()));
            System.out.println("------------------------------------------------------"+"from "+ Integer.toString(currentPage-1));
            SearchHit[] hits = response.getHits().getHits();
            for (SearchHit hit:hits) {
                System.out.println(hit.getSourceAsString());
            }

            if(hits.length==0){//返回没值时，则表示遍历完成
                hasMore=false;
            }
            currentPage++;
            searchSourceBuilder.from((currentPage-1)*pageSize);
            searchSourceBuilder.size(pageSize);
        }
        System.out.println("全部查完");

    }

浅分页多次查询

16、深分页(scroll【还有一种search_after方法】)

 //多次分页查询（scroll）
    @Test
    public void testSearchByPage3() throws IOException {
        //Integer currentPage=1;
        Integer pageSize=3;

        SearchRequest searchRequest=new SearchRequest();
        searchRequest.indices("kuangshen");
        searchRequest.scroll(TimeValue.timeValueMinutes(1L));//设置scroll失效时间为1分钟
        SearchSourceBuilder searchSourceBuilder=new SearchSourceBuilder();
        //不需要传从第几条开始
        //searchSourceBuilder.from(currentPage-1);
        searchSourceBuilder.size(pageSize);
        searchSourceBuilder.sort("age", SortOrder.ASC);//排序，查出来的数据根据age排序
        searchRequest.source(searchSourceBuilder);
        SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT);
        //System.out.println(JSON.toJSONString(response.getHits()));
        System.out.println("------------------------首页------------------------------");
        SearchHit[] hits = response.getHits().getHits();
        for (SearchHit hit:hits) {
            System.out.println(hit.getSourceAsString());
        }

        String scrollId=response.getScrollId();
        System.out.println("scrollId为： "+response.getScrollId());

        Boolean hasMore=true;
        while (hasMore){
            SearchScrollRequest searchScrollRequest=new SearchScrollRequest();
            searchScrollRequest.scroll(TimeValue.timeValueMinutes(1L));
            searchScrollRequest.scrollId(scrollId);
            SearchResponse scrollResponse = client.scroll(searchScrollRequest, RequestOptions.DEFAULT);
            System.out.println("------------------------下一页------------------------------");
            SearchHit[] hits1 = scrollResponse.getHits().getHits();
            for (SearchHit hit:hits1) {
                System.out.println(hit.getSourceAsString());
            }
            if(hits1.length==0){//返回没值时，则表示遍历完成
                hasMore=false;
            }
            scrollId = scrollResponse.getScrollId();
            System.out.println("scrollId为： "+scrollId);
        }
        System.out.println("全部查完");
    }

深分页查询结果

16、from+size 和 scroll两种方式比较

分页方式比较

scroll用的是快照模式，有个窗口期，都是基于这个窗口期的快照来做的查询，scrollId对应的就是这个快照，scrollId是不变的

17、elasticsearch scroll查询的原理

1、https://elasticsearch.cn/question/2935
2、https://www.elastic.co/guide/cn/elasticsearch/guide/2.x/_fetch_phase.html
3、https://www.jianshu.com/p/91d03b16af77