elasticsearch实现类似京东的商品搜索效果（elasticsearch动态聚合）

小灰灰 2023-07-24 08:49 70阅读 0赞

用到京东的对其搜索应该不会陌生，其搜索也是使用elasticsearch完成的，下图为一个搜索效果图：

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2pvaG4xMzM3_size_16_color_FFFFFF_t_70][]

搜索筛选条件会根据查询返回的结果动态变化，要实现这个功能就要用到elasticsearch的聚合功能，先看下商品索引对应的映射：

{
      "mapping": {
        "es-product": {
          "dynamic_templates": [
            {
              "strings_as_keyword": {
                "match_mapping_type": "string",
                "mapping": {
                  "type": "keyword"
                }
              }
            }
          ],
          "properties": {
            "aggProperties": {
              "type": "nested",
              "properties": {
                "key": {
                  "type": "keyword"
                },
                "value": {
                  "type": "keyword"
                }
              }
            },
            "brandName": {
              "type": "keyword"
            },
            "id": {
              "type": "keyword"
            },
            "level1Category": {
              "type": "keyword"
            },
            "level2Category": {
              "type": "keyword"
            },
            "level3Category": {
              "type": "keyword"
            },
            "mainTitle": {
              "type": "text",
              "analyzer": "ik_max_word"
            },
            "searchProperties": {
              "properties": {
                "光泽度": {
                  "type": "keyword"
                },
                "光源个数": {
                  "type": "keyword"
                },
                "光源类型": {
                  "type": "keyword"
                },
                "功率": {
                  "type": "keyword"
                },
                "包装体积": {
                  "type": "keyword"
                },
                "型号": {
                  "type": "keyword"
                },
                "密度": {
                  "type": "keyword"
                },
                "导热性": {
                  "type": "keyword"
                },
                "导电性": {
                  "type": "keyword"
                },
                "延展性": {
                  "type": "keyword"
                },
                "抗氧化性": {
                  "type": "keyword"
                },
                "是否带护栏": {
                  "type": "keyword"
                },
                "是否带软靠": {
                  "type": "keyword"
                },
                "是否带遥控器": {
                  "type": "keyword"
                },
                "照射面积": {
                  "type": "keyword"
                },
                "熔点": {
                  "type": "keyword"
                },
                "磁性": {
                  "type": "keyword"
                },
                "表面机理": {
                  "type": "keyword"
                },
                "适用人数": {
                  "type": "keyword"
                },
                "适用空间": {
                  "type": "keyword"
                },
                "风格": {
                  "type": "keyword"
                }
              }
            },
            "subTitle": {
              "type": "keyword"
            }
          }
        }
      }
    }

searchProperties部分为动态属性，使用elasticsearch的dynamic template配置，aggProperties部分为动态聚合所用，通过aggProperties下面的值动态聚合满足条件的搜索结果所具有的所有属性，比如光泽度、熔点等，而searchProperties是为搜索使用，先说下属性动态聚合的实现，下面是elasticsearch的查询脚本：

{
      "from" : 0, "size" : 100,
      "query": {
        "bool":{
          "must":[{
            "match" : {
              "mainTitle" : "拉手"
            }
          }
          ]
        }
      },
      "aggs": {
        "dynamicproperty": {
          "nested": {
            "path": "aggProperties"
          },
          "aggs": {
            "name": {
              "terms": {
                "field": "aggProperties.key"
              },
              "aggs": {
                "value": {
                  "terms": {
                    "field": "aggProperties.value"
                  }
                }
              }
            }
          }
        }
      }
    }

通过上面的搜索得到下面结果：

{
        "took": 13,
        "timed_out": false,
        "_shards": {
            "total": 3,
            "successful": 3,
            "skipped": 0,
            "failed": 0
        },
        "hits": {
            "total": 8,
            "max_score": 1.3591974,
            "hits": [
                {
                    "_index": "smp-product",
                    "_type": "es-product",
                    "_id": "2c95ae137157829c017157c3c61c0089",
                    "_score": 1.3591974,
                    "_source": {
                        "id": "2c95ae137157829c017157c3c61c0089",
                        "mainTitle": "富达铭黑色隐形拉手免打孔铝合金拉手一字型暗拉手橱柜衣柜门长把手隐藏拉手长拉手 咖啡150mm",
                        "subTitle": "",
                        "brandName": "三叶",
                        "aggProperties": [
                            {
                                "key": "密度",
                                "value": "2000克/cm³"
                            },
                            {
                                "key": "熔点",
                                "value": "2000℃"
                            },
                            {
                                "key": "导热性",
                                "value": "很好"
                            },
                            {
                                "key": "延展性",
                                "value": "易锻物质，不需退火可锤炼可压延"
                            },
                            {
                                "key": "导电性",
                                "value": "是"
                            },
                            {
                                "key": "磁性",
                                "value": "顺磁材料"
                            },
                            {
                                "key": "光泽度",
                                "value": "哑光"
                            },
                            {
                                "key": "表面机理",
                                "value": "拉丝"
                            },
                            {
                                "key": "抗氧化性",
                                "value": "中"
                            }
                        ],
                        "searchProperties": {
                            "表面机理": "拉丝",
                            "抗氧化性": "中",
                            "导热性": "很好",
                            "延展性": "易锻物质，不需退火可锤炼可压延",
                            "磁性": "顺磁材料",
                            "导电性": "是",
                            "密度": "2000克/cm³",
                            "熔点": "2000℃",
                            "光泽度": "哑光"
                        }
                    }
                },
                {
                    "_index": "smp-product",
                    "_type": "es-product",
                    "_id": "2c95ae137157829c017157ad040c004d",
                    "_score": 1.2214447,
                    "_source": {
                        "id": "2c95ae137157829c017157ad040c004d",
                        "mainTitle": "致诺 老式铁皮拉手 不锈钢拉手大木门蟹壳拉手 明装四孔铁皮门轻把手简易拉手老式通用型防盗门把手 大号2个价格",
                        "subTitle": "",
                        "brandName": "三叶",
                        "aggProperties": [
                            {
                                "key": "密度",
                                "value": "1000克/cm³"
                            },
                            {
                                "key": "熔点",
                                "value": "1000℃"
                            },
                            {
                                "key": "导热性",
                                "value": "较好"
                            },
                            {
                                "key": "延展性",
                                "value": "易锻物质，不需退火可锤炼可压延"
                            }
                        ],
                        "searchProperties": {
                            "导热性": "较好",
                            "延展性": "易锻物质，不需退火可锤炼可压延",
                            "密度": "1000克/cm³",
                            "熔点": "1000℃"
                        }
                    }
                },
                {
                    "_index": "smp-product",
                    "_type": "es-product",
                    "_id": "2c95ae137157829c017157a991b60011",
                    "_score": 1.1077276,
                    "_source": {
                        "id": "2c95ae137157829c017157a991b60011",
                        "mainTitle": "卡贝拉手衣柜门拉手抽屉橱柜衣柜鞋柜高档门把手现代简约五金配件",
                        "subTitle": "",
                        "brandName": "宗艺石材",
                        "aggProperties": [
                            {
                                "key": "密度",
                                "value": "1000克/cm³"
                            },
                            {
                                "key": "熔点",
                                "value": "2000℃"
                            },
                            {
                                "key": "导热性",
                                "value": "较好"
                            },
                            {
                                "key": "导电性",
                                "value": "否"
                            },
                            {
                                "key": "磁性",
                                "value": "顺磁材料"
                            },
                            {
                                "key": "光泽度",
                                "value": "哑光"
                            },
                            {
                                "key": "表面机理",
                                "value": "拉丝"
                            },
                            {
                                "key": "抗氧化性",
                                "value": "中"
                            }
                        ],
                        "searchProperties": {
                            "表面机理": "拉丝",
                            "抗氧化性": "中",
                            "导热性": "较好",
                            "磁性": "顺磁材料",
                            "导电性": "否",
                            "密度": "1000克/cm³",
                            "熔点": "2000℃",
                            "光泽度": "哑光"
                        }
                    }
                },
                {
                    "_index": "smp-product",
                    "_type": "es-product",
                    "_id": "2c95ae137157829c017157c5c98100a7",
                    "_score": 0.77200073,
                    "_source": {
                        "id": "2c95ae137157829c017157c5c98100a7",
                        "mainTitle": "CD超代欧式简约陶瓷衣柜柜门抽屉把手小拉手 田园大理石陶瓷酒柜拉手 单只价格 A7-96孔距-长120mm",
                        "subTitle": "",
                        "brandName": "三叶",
                        "aggProperties": [
                            {
                                "key": "密度",
                                "value": "1500克/cm³"
                            },
                            {
                                "key": "熔点",
                                "value": "1600℃"
                            },
                            {
                                "key": "导热性",
                                "value": "较好"
                            },
                            {
                                "key": "延展性",
                                "value": "易锻物质，不需退火可锤炼可压延"
                            },
                            {
                                "key": "导电性",
                                "value": "是"
                            },
                            {
                                "key": "磁性",
                                "value": "顺磁材料"
                            },
                            {
                                "key": "光泽度",
                                "value": "哑光"
                            },
                            {
                                "key": "表面机理",
                                "value": "拉丝"
                            },
                            {
                                "key": "抗氧化性",
                                "value": "高"
                            }
                        ],
                        "searchProperties": {
                            "表面机理": "拉丝",
                            "抗氧化性": "高",
                            "导热性": "较好",
                            "延展性": "易锻物质，不需退火可锤炼可压延",
                            "磁性": "顺磁材料",
                            "导电性": "是",
                            "密度": "1500克/cm³",
                            "熔点": "1600℃",
                            "光泽度": "哑光"
                        }
                    }
                },
                {
                    "_index": "smp-product",
                    "_type": "es-product",
                    "_id": "2c95ae137157829c017157aaa03d0022",
                    "_score": 0.7351246,
                    "_source": {
                        "id": "2c95ae137157829c017157aaa03d0022",
                        "mainTitle": "适用于步阳群升新多星月神春天进户大门防盗门把手 拉手面板双快加厚实",
                        "subTitle": "",
                        "brandName": "西门子",
                        "aggProperties": [
                            {
                                "key": "密度",
                                "value": "1000克/cm³"
                            },
                            {
                                "key": "熔点",
                                "value": "1500℃"
                            },
                            {
                                "key": "导热性",
                                "value": "较好"
                            },
                            {
                                "key": "延展性",
                                "value": "易锻物质，不需退火可锤炼可压延"
                            }
                        ],
                        "searchProperties": {
                            "导热性": "较好",
                            "延展性": "易锻物质，不需退火可锤炼可压延",
                            "密度": "1000克/cm³",
                            "熔点": "1500℃"
                        }
                    }
                },
                {
                    "_index": "smp-product",
                    "_type": "es-product",
                    "_id": "2c95ae137157829c017157ac51f9003f",
                    "_score": 0.6896249,
                    "_source": {
                        "id": "2c95ae137157829c017157ac51f9003f",
                        "mainTitle": "防盗门把手拉手进户门拉手双活双快执手手柄加厚大门锁具配件 加厚铝合金门把手 >55mm 通用型 不带钥匙",
                        "subTitle": "",
                        "brandName": "西门子",
                        "aggProperties": [
                            {
                                "key": "密度",
                                "value": "1500克/cm³"
                            },
                            {
                                "key": "熔点",
                                "value": "2000℃"
                            },
                            {
                                "key": "导热性",
                                "value": "较好"
                            },
                            {
                                "key": "延展性",
                                "value": "易锻物质，不需退火可锤炼可压延"
                            },
                            {
                                "key": "导电性",
                                "value": "是"
                            },
                            {
                                "key": "磁性",
                                "value": "顺磁材料"
                            },
                            {
                                "key": "光泽度",
                                "value": "高光"
                            },
                            {
                                "key": "表面机理",
                                "value": "拉丝"
                            },
                            {
                                "key": "抗氧化性",
                                "value": "中"
                            }
                        ],
                        "searchProperties": {
                            "表面机理": "拉丝",
                            "抗氧化性": "中",
                            "导热性": "较好",
                            "延展性": "易锻物质，不需退火可锤炼可压延",
                            "磁性": "顺磁材料",
                            "导电性": "是",
                            "密度": "1500克/cm³",
                            "熔点": "2000℃",
                            "光泽度": "高光"
                        }
                    }
                },
                {
                    "_index": "smp-product",
                    "_type": "es-product",
                    "_id": "2c95ae137157829c017157add2c8005c",
                    "_score": 0.4982656,
                    "_source": {
                        "id": "2c95ae137157829c017157add2c8005c",
                        "mainTitle": "汇乐斯 防盗门把手拉手执手大门把手门把手门锁把手锁体锁帽 面板把手一对/配件+外固定把手 左开",
                        "subTitle": "",
                        "brandName": "三棵树",
                        "aggProperties": [
                            {
                                "key": "密度",
                                "value": "1000克/cm³"
                            },
                            {
                                "key": "熔点",
                                "value": "1000℃"
                            },
                            {
                                "key": "导热性",
                                "value": "很好"
                            },
                            {
                                "key": "延展性",
                                "value": "易锻物质，不需退火可锤炼可压延"
                            }
                        ],
                        "searchProperties": {
                            "导热性": "很好",
                            "延展性": "易锻物质，不需退火可锤炼可压延",
                            "密度": "1000克/cm³",
                            "熔点": "1000℃"
                        }
                    }
                },
                {
                    "_index": "smp-product",
                    "_type": "es-product",
                    "_id": "2c95ae137157829c017157c4cb880098",
                    "_score": 0.48578173,
                    "_source": {
                        "id": "2c95ae137157829c017157c4cb880098",
                        "mainTitle": "防盗门把手拉手双活双快执手手柄大门锁配件铝合金门把手 加厚白金带点固定款(13*13) >55mm 通用型 不带钥匙",
                        "subTitle": "",
                        "brandName": "三叶",
                        "aggProperties": [
                            {
                                "key": "密度",
                                "value": "1500克/cm³"
                            },
                            {
                                "key": "熔点",
                                "value": "1500℃"
                            },
                            {
                                "key": "导热性",
                                "value": "较好"
                            },
                            {
                                "key": "延展性",
                                "value": "易锻物质，不需退火可锤炼可压延"
                            },
                            {
                                "key": "导电性",
                                "value": "否"
                            },
                            {
                                "key": "磁性",
                                "value": "铁磁材料"
                            },
                            {
                                "key": "光泽度",
                                "value": "高光"
                            },
                            {
                                "key": "表面机理",
                                "value": "拉丝"
                            },
                            {
                                "key": "抗氧化性",
                                "value": "中"
                            }
                        ],
                        "searchProperties": {
                            "表面机理": "拉丝",
                            "抗氧化性": "中",
                            "导热性": "较好",
                            "延展性": "易锻物质，不需退火可锤炼可压延",
                            "磁性": "铁磁材料",
                            "导电性": "否",
                            "密度": "1500克/cm³",
                            "熔点": "1500℃",
                            "光泽度": "高光"
                        }
                    }
                }
            ]
        },
        "aggregations": {
            "dynamicproperty": {
                "doc_count": 56,
                "name": {
                    "doc_count_error_upper_bound": 0,
                    "sum_other_doc_count": 0,
                    "buckets": [
                        {
                            "key": "密度",
                            "doc_count": 8,
                            "value": {
                                "doc_count_error_upper_bound": 0,
                                "sum_other_doc_count": 0,
                                "buckets": [
                                    {
                                        "key": "1000克/cm³",
                                        "doc_count": 4
                                    },
                                    {
                                        "key": "1500克/cm³",
                                        "doc_count": 3
                                    },
                                    {
                                        "key": "2000克/cm³",
                                        "doc_count": 1
                                    }
                                ]
                            }
                        },
                        {
                            "key": "导热性",
                            "doc_count": 8,
                            "value": {
                                "doc_count_error_upper_bound": 0,
                                "sum_other_doc_count": 0,
                                "buckets": [
                                    {
                                        "key": "较好",
                                        "doc_count": 6
                                    },
                                    {
                                        "key": "很好",
                                        "doc_count": 2
                                    }
                                ]
                            }
                        },
                        {
                            "key": "熔点",
                            "doc_count": 8,
                            "value": {
                                "doc_count_error_upper_bound": 0,
                                "sum_other_doc_count": 0,
                                "buckets": [
                                    {
                                        "key": "2000℃",
                                        "doc_count": 3
                                    },
                                    {
                                        "key": "1000℃",
                                        "doc_count": 2
                                    },
                                    {
                                        "key": "1500℃",
                                        "doc_count": 2
                                    },
                                    {
                                        "key": "1600℃",
                                        "doc_count": 1
                                    }
                                ]
                            }
                        },
                        {
                            "key": "延展性",
                            "doc_count": 7,
                            "value": {
                                "doc_count_error_upper_bound": 0,
                                "sum_other_doc_count": 0,
                                "buckets": [
                                    {
                                        "key": "易锻物质，不需退火可锤炼可压延",
                                        "doc_count": 7
                                    }
                                ]
                            }
                        },
                        {
                            "key": "光泽度",
                            "doc_count": 5,
                            "value": {
                                "doc_count_error_upper_bound": 0,
                                "sum_other_doc_count": 0,
                                "buckets": [
                                    {
                                        "key": "哑光",
                                        "doc_count": 3
                                    },
                                    {
                                        "key": "高光",
                                        "doc_count": 2
                                    }
                                ]
                            }
                        },
                        {
                            "key": "导电性",
                            "doc_count": 5,
                            "value": {
                                "doc_count_error_upper_bound": 0,
                                "sum_other_doc_count": 0,
                                "buckets": [
                                    {
                                        "key": "是",
                                        "doc_count": 3
                                    },
                                    {
                                        "key": "否",
                                        "doc_count": 2
                                    }
                                ]
                            }
                        },
                        {
                            "key": "抗氧化性",
                            "doc_count": 5,
                            "value": {
                                "doc_count_error_upper_bound": 0,
                                "sum_other_doc_count": 0,
                                "buckets": [
                                    {
                                        "key": "中",
                                        "doc_count": 4
                                    },
                                    {
                                        "key": "高",
                                        "doc_count": 1
                                    }
                                ]
                            }
                        },
                        {
                            "key": "磁性",
                            "doc_count": 5,
                            "value": {
                                "doc_count_error_upper_bound": 0,
                                "sum_other_doc_count": 0,
                                "buckets": [
                                    {
                                        "key": "顺磁材料",
                                        "doc_count": 4
                                    },
                                    {
                                        "key": "铁磁材料",
                                        "doc_count": 1
                                    }
                                ]
                            }
                        },
                        {
                            "key": "表面机理",
                            "doc_count": 5,
                            "value": {
                                "doc_count_error_upper_bound": 0,
                                "sum_other_doc_count": 0,
                                "buckets": [
                                    {
                                        "key": "拉丝",
                                        "doc_count": 5
                                    }
                                ]
                            }
                        }
                    ]
                }
            }
        }
    }

这样就在返回搜索结果的同时返回了满足条件的所有索引的动态属性，把aggregations中的数据处理后返回给前端就可以实现类似京东的商品搜索效果。

接下来给出基于RestHighLevelClient的客户端搜索实现：

@Test
        public void aggregate2() throws IOException {
            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
            BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
            boolQueryBuilder.must(QueryBuilders.matchQuery("mainTitle","拉手"));
    
            BoolQueryBuilder boolQueryBuilder1 = QueryBuilders.boolQuery();
            boolQueryBuilder1.must(QueryBuilders.matchQuery("props.key","密度"));
            boolQueryBuilder1.must(QueryBuilders.matchQuery("props.value","2000克/cm³"));
            NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("props",boolQueryBuilder1, ScoreMode.Total);
            searchSourceBuilder.query(nestedQueryBuilder);
            NestedAggregationBuilder level1Aggs = AggregationBuilders.nested("level1Agg","props");
            TermsAggregationBuilder propKeyAgg = AggregationBuilders.terms("keyAggs").field("props.key");
            level1Aggs.subAggregation(propKeyAgg);
            TermsAggregationBuilder propValAgg = AggregationBuilders.terms("valAggs").field("props.value");
            propKeyAgg.subAggregation(propValAgg);
            searchSourceBuilder.aggregation(level1Aggs);
            SearchRequest searchRequest = new SearchRequest();
            searchRequest.indices("smp-product");
            searchRequest.source(searchSourceBuilder);
            SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
            if(RestStatus.OK == searchResponse.status()){
                //查询结果
                SearchHits hits = searchResponse.getHits();
                log.info("hits={}",hits.getTotalHits());
                SearchHit[] arrs = hits.getHits();
                List<ESProduct> prods = new ArrayList<>(arrs.length);
                for(int i=0,len=arrs.length;i<len;i++){
                    SearchHit hit = arrs[i];
                    //ESProduct为商品类
                    ESProduct prod = JSON.parseObject(hit.getSourceAsString(),ESProduct.class);
                    prods.add(prod);
                }
                Aggregations aggregations = searchResponse.getAggregations();
                //聚合结果处理
                ParsedNested level1Agg = aggregations.get("level1Agg");
                Aggregations keyAggs = level1Agg.getAggregations();
                Terms byAgeAggregation = keyAggs.get("keyAggs");
                List<FilterParamsDto> params = new ArrayList<>();
                for(Terms.Bucket buck : byAgeAggregation.getBuckets()) {
                    Aggregations valAggs =  buck.getAggregations();
                    Terms byValAggs =  valAggs.get("valAggs");
                    FilterParamsDto param = new FilterParamsDto();
                    param.setName(buck.getKeyAsString());
                    List<String> vals = new ArrayList<>();
                    param.setFilters(vals);
                    for(Terms.Bucket bk : byValAggs.getBuckets()){
                        log.info("key: " + bk.getKeyAsString());
                        vals.add(bk.getKeyAsString());
                    }
                    params.add(param);
                }
            }
            log.info("finish");
        }

上面程序对应的ESProduct类定义如下：

@Getter
    @Setter
    @Document(indexName="smp-product",type="es-product",shards=3,replicas = 0)
    @Mapping(mappingPath="mapping/es-product.json")
    public class ESProduct extends BaseEntityMethods {
    
        @Field
        String id;
    
        /**
         * 商品名字
         */
        @Field(name="mainTitle")
        String mainTitle;
    
        @Field(name="subTitle")
        String subTitle;
    
        /**
         * 商品品牌
         */
        @Field(name="brandName")
        String brandName;
    
        /**
         * 一级类目名称
         */
        @JsonIgnore
        @Field(name="level1Category")
        String level1Category;
    
        /**
         * 二级类目名称
         */
        @JsonIgnore
        @Field(name="level2Category")
        String level2Category;
    
        /**
         * 三级类目名称
         */
        @JsonIgnore
        @Field(name="level3Category")
        String level3Category;
    
        /**
         * 聚合关键词使用
         */
        @Field(name="aggProperties")
        List<SearchProperty> aggProperties;
    
        /**
         * 检索使用
         */
        @Field(name="searchProperties")
        Map<String,String> searchProperties;
    }

PS：

1、本文使用的elasticsearch为6.8版本

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L2pvaG4xMzM3_size_16_color_FFFFFF_t_70]: /images/20230528/a704837662a14013a97eeef083895b9b.png