caojiahao/dev_caojiahao
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 优化es搜索，增加如搜索超过10000条信息的处理

Browse Source
This commit is contained in:
misworga831 2023-05-09 13:59:23 +08:00
parent c286d05ad8
commit ae93661763
3 changed files with 27 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
framework/src/main/java/cn/lili/elasticsearch/BaseElasticsearchService.java
View File

@ -90,6 +90,7 @@ public abstract class BaseElasticsearchService {
request.settings(Settings.builder()
.put("index.number_of_shards", elasticsearchProperties.getIndex().getNumberOfShards())
.put("index.number_of_replicas", elasticsearchProperties.getIndex().getNumberOfReplicas())
.put("index.max_result_window", 100000) //最大查询结果数
.put("index.mapping.total_fields.limit", 2000));
//创建索引

31
framework/src/main/java/cn/lili/modules/search/serviceimpl/EsGoodsIndexServiceImpl.java
View File

@ -66,6 +66,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
import org.springframework.data.elasticsearch.core.SearchHit;
import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.data.elasticsearch.core.SearchPage;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.stereotype.Service;
@ -178,7 +179,7 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
skuCountQueryWrapper.eq("auth_flag", GoodsAuthEnum.PASS.name());
skuCountQueryWrapper.eq("market_enable", GoodsStatusEnum.UPPER.name());
skuCountQueryWrapper.eq("delete_flag", false);
skuCountQueryWrapper.ge("quantity", 0);
skuCountQueryWrapper.gt("quantity", 0);
resultMap = new HashMap<>();
resultMap.put(KEY_SUCCESS, 0L);
resultMap.put(KEY_FAIL, 0L);
@ -675,14 +676,30 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
@Override
public void deleteEsGoodsPromotionByPromotionKey(String promotionsKey) {
ThreadUtil.execAsync(() -> {
BulkRequest bulkRequest = new BulkRequest();
for (EsGoodsIndex goodsIndex : this.goodsIndexRepository.findAll()) {
UpdateRequest updateRequest = this.removePromotionByPromotionKey(goodsIndex, promotionsKey);
if (updateRequest != null) {
bulkRequest.add(updateRequest);
for (int i = 0; ; i++) {
BulkRequest bulkRequest = new BulkRequest();
NativeSearchQueryBuilder nativeSearchQueryBuilder = new NativeSearchQueryBuilder();
nativeSearchQueryBuilder.withQuery(QueryBuilders.matchAllQuery());
nativeSearchQueryBuilder.withPageable(PageRequest.of(i, 1000));
try {
SearchHits<EsGoodsIndex> esGoodsIndices = this.restTemplate.search(nativeSearchQueryBuilder.build(), EsGoodsIndex.class);
if (esGoodsIndices.isEmpty() || esGoodsIndices.getSearchHits().isEmpty()) {
break;
}
for (SearchHit<EsGoodsIndex> searchHit : esGoodsIndices.getSearchHits()) {
EsGoodsIndex goodsIndex = searchHit.getContent();
UpdateRequest updateRequest = this.removePromotionByPromotionKey(goodsIndex, promotionsKey);
if (updateRequest != null) {
bulkRequest.add(updateRequest);
}
}
this.executeBulkUpdateRequest(bulkRequest);
} catch (Exception e) {
log.error("删除索引中指定的促销活动id的促销活动失败！key: {}", promotionsKey, e);
return;
}
}
this.executeBulkUpdateRequest(bulkRequest);
});
}

3
framework/src/main/java/cn/lili/modules/search/serviceimpl/EsGoodsSearchServiceImpl.java
View File

@ -84,11 +84,12 @@ public class EsGoodsSearchServiceImpl implements EsGoodsSearchService {
//如果搜索词不为空，且明显不是sql注入，那么就将搜索词加入热搜词
//PS:线上环境运行很多客户反馈被sql攻击，写在了搜索热词里，这里控制命中关键字就不做热词统计，如果线上比较严格可以调用关键词替换，不过不建议这么做
if (CharSequenceUtil.isNotBlank(searchDTO.getKeyword()) && !SqlFilter.hit(searchDTO.getKeyword())) {
if (CharSequenceUtil.isNotBlank(searchDTO.getKeyword()) && Boolean.FALSE.equals(SqlFilter.hit(searchDTO.getKeyword()))) {
cache.incrementScore(CachePrefix.HOT_WORD.getPrefix(), searchDTO.getKeyword());
}
NativeSearchQueryBuilder searchQueryBuilder = createSearchQueryBuilder(searchDTO, pageVo);
NativeSearchQuery searchQuery = searchQueryBuilder.build();
searchQuery.setTrackTotalHits(true);
log.debug("searchGoods DSL:{}", searchQuery.getQuery());
SearchHits<EsGoodsIndex> search = restTemplate.search(searchQuery, EsGoodsIndex.class);
return SearchHitSupport.searchPageFor(search, searchQuery.getPageable());