pref: 优化es生成全部商品索引,减少数据库操作

This commit is contained in:
paulGao 2022-11-02 19:43:17 +08:00
parent ca23b5ab33
commit 3b078cfd6b
12 changed files with 189 additions and 61 deletions

View File

@ -8,6 +8,7 @@ import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.service.IService;
import java.util.List;
import java.util.Map;
/**
* 商品品牌业务层
@ -40,6 +41,14 @@ public interface BrandService extends IService<Brand> {
*/
List<Brand> getBrandsByCategory(String categoryId);
/**
* 根据分类ID获取品牌列表
*
* @param categoryIds 分类ID
* @return 品牌列表
*/
List<Map<String, Object>> getBrandsMapsByCategory(List<String> categoryIds, String columns);
/**
* 添加品牌
*

View File

@ -6,6 +6,7 @@ import cn.lili.modules.goods.entity.vos.CategoryVO;
import com.baomidou.mybatisplus.extension.service.IService;
import java.util.List;
import java.util.Map;
/**
* 商品分类业务层
@ -41,6 +42,14 @@ public interface CategoryService extends IService<Category> {
*/
List<Category> listByIdsOrderByLevel(List<String> ids);
/**
* 根据分类id集合获取所有分类根据层级排序
*
* @param ids 分类ID集合
* @return 商品分类列表
*/
List<Map<String, Object>> listMapsByIdsOrderByLevel(List<String> ids, String columns);
/**
* 获取分类树
*

View File

@ -5,6 +5,7 @@ import cn.lili.modules.goods.entity.vos.StoreGoodsLabelVO;
import com.baomidou.mybatisplus.extension.service.IService;
import java.util.List;
import java.util.Map;
/**
* 店铺商品分类业务层
@ -30,6 +31,14 @@ public interface StoreGoodsLabelService extends IService<StoreGoodsLabel> {
*/
List<StoreGoodsLabel> listByStoreIds(List<String> ids);
/**
* 根据分类id集合获取所有店铺分类根据层级排序
*
* @param ids 商家ID
* @return 店铺分类列表
*/
List<Map<String, Object>> listMapsByStoreIds(List<String> ids, String columns);
/**
* 添加商品分类
*

View File

@ -23,6 +23,7 @@ import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@ -68,6 +69,14 @@ public class BrandServiceImpl extends ServiceImpl<BrandMapper, Brand> implements
return new ArrayList<>();
}
@Override
public List<Map<String, Object>> getBrandsMapsByCategory(List<String> categoryIds, String columns) {
QueryWrapper<Brand> queryWrapper = new QueryWrapper<>();
queryWrapper.select(columns);
queryWrapper.in("id", categoryIds);
return this.listMaps(queryWrapper);
}
@Override
public boolean addBrand(BrandVO brandVO) {

View File

@ -25,10 +25,7 @@ import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.*;
import java.util.stream.Collectors;
@ -80,6 +77,14 @@ public class CategoryServiceImpl extends ServiceImpl<CategoryMapper, Category> i
return this.list(new LambdaQueryWrapper<Category>().in(Category::getId, ids).orderByAsc(Category::getLevel));
}
@Override
public List<Map<String, Object>> listMapsByIdsOrderByLevel(List<String> ids, String columns) {
QueryWrapper<Category> queryWrapper = new QueryWrapper<>();
queryWrapper.select(columns);
queryWrapper.in("id", ids).orderByAsc("level");
return this.listMaps(queryWrapper);
}
@Override
public List<CategoryVO> categoryTree() {
List<CategoryVO> categoryVOList = (List<CategoryVO>) cache.get(CachePrefix.CATEGORY.getPrefix());

View File

@ -12,6 +12,7 @@ import cn.lili.modules.goods.entity.vos.StoreGoodsLabelVO;
import cn.lili.modules.goods.mapper.StoreGoodsLabelMapper;
import cn.lili.modules.goods.service.StoreGoodsLabelService;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
@ -22,6 +23,7 @@ import org.springframework.transaction.annotation.Transactional;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
/**
* 店铺商品分类业务层实现
@ -83,6 +85,13 @@ public class StoreGoodsLabelServiceImpl extends ServiceImpl<StoreGoodsLabelMappe
return this.list(new LambdaQueryWrapper<StoreGoodsLabel>().in(StoreGoodsLabel::getId, ids).orderByAsc(StoreGoodsLabel::getLevel));
}
@Override
public List<Map<String, Object>> listMapsByStoreIds(List<String> ids, String columns) {
QueryWrapper<StoreGoodsLabel> queryWrapper = new QueryWrapper<StoreGoodsLabel>().in("id", ids).orderByAsc("level");
queryWrapper.select(columns);
return this.listMaps(queryWrapper);
}
@Override
@Transactional(rollbackFor = Exception.class)
public StoreGoodsLabel addStoreGoodsLabel(StoreGoodsLabel storeGoodsLabel) {

View File

@ -4,6 +4,7 @@ import cn.lili.cache.CachePrefix;
import cn.lili.common.enums.PromotionTypeEnum;
import cn.lili.common.vo.PageVO;
import cn.lili.modules.goods.entity.dos.GoodsSku;
import cn.lili.modules.goods.entity.dto.GoodsSkuDTO;
import cn.lili.modules.promotion.entity.dos.PromotionGoods;
import cn.lili.modules.promotion.entity.dto.search.PromotionGoodsSearchParams;
import com.baomidou.mybatisplus.core.metadata.IPage;
@ -43,6 +44,15 @@ public interface PromotionGoodsService extends IService<PromotionGoods> {
*/
List<PromotionGoods> findSkuValidPromotion(String skuId, String storeIds);
/**
* 获取sku所有有效活动
*
* @param skus 商品skuId
* @return 促销商品集合
*/
List<PromotionGoods> findSkuValidPromotions(List<GoodsSkuDTO> skus);
/**
* 分页获取促销商品信息
*

View File

@ -35,4 +35,12 @@ public interface PromotionService {
* @param goodsIdsJsonStr
*/
void removeByGoodsIds(String goodsIdsJsonStr);
/**
* 根据促销商品信息包装促销信息
*
* @param promotionGoodsList 促销商品信息
* @return 促销信息
*/
Map<String, Object> wrapperPromotionMapList(List<PromotionGoods> promotionGoodsList);
}

View File

@ -7,6 +7,7 @@ import cn.hutool.json.JSONUtil;
import cn.lili.common.enums.PromotionTypeEnum;
import cn.lili.common.vo.PageVO;
import cn.lili.modules.goods.entity.dos.GoodsSku;
import cn.lili.modules.goods.entity.dto.GoodsSkuDTO;
import cn.lili.modules.goods.entity.vos.GoodsVO;
import cn.lili.modules.goods.service.GoodsService;
import cn.lili.modules.goods.service.GoodsSkuService;
@ -35,6 +36,7 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.*;
import java.util.stream.Collectors;
/**
* 促销商品业务层实现
@ -87,6 +89,22 @@ public class PromotionGoodsServiceImpl extends ServiceImpl<PromotionGoodsMapper,
return this.list(queryWrapper);
}
@Override
public List<PromotionGoods> findSkuValidPromotions(List<GoodsSkuDTO> skus) {
List<String> categories = skus.stream().map(GoodsSku::getCategoryPath).collect(Collectors.toList());
List<String> skuIds = skus.stream().map(GoodsSku::getId).collect(Collectors.toList());
List<String> categoriesPath = new ArrayList<>();
categories.forEach(i -> categoriesPath.addAll(Arrays.asList(i.split(","))));
QueryWrapper<PromotionGoods> queryWrapper = new QueryWrapper<>();
queryWrapper.and(i -> i.or(j -> j.in(SKU_ID_COLUMN, skuIds))
.or(n -> n.eq("scope_type", PromotionsScopeTypeEnum.ALL.name()))
.or(n -> n.and(k -> k.eq("scope_type", PromotionsScopeTypeEnum.PORTION_GOODS_CATEGORY.name())
.and(l -> l.in("scope_id", categoriesPath)))));
queryWrapper.and(i -> i.or(PromotionTools.queryPromotionStatus(PromotionsStatusEnum.START)).or(PromotionTools.queryPromotionStatus(PromotionsStatusEnum.NEW)));
return this.list(queryWrapper);
}
@Override
public IPage<PromotionGoods> pageFindAll(PromotionGoodsSearchParams searchParams, PageVO pageVo) {
return this.page(PageUtil.initPage(pageVo), searchParams.queryWrapper());

View File

@ -89,8 +89,19 @@ public class PromotionServiceImpl implements PromotionService {
*/
public Map<String, Object> getGoodsSkuPromotionMap(String storeId, String goodsSkuId) {
String storeIds = storeId + "," + PromotionTools.PLATFORM_ID;
Map<String, Object> promotionMap = new HashMap<>();
List<PromotionGoods> promotionGoodsList = promotionGoodsService.findSkuValidPromotion(goodsSkuId, storeIds);
return wrapperPromotionMapList(promotionGoodsList);
}
@Override
public void removeByGoodsIds(String goodsIdsJsonStr) {
List<String> goodsIds = JSONUtil.toList(goodsIdsJsonStr, String.class);
promotionGoodsService.deletePromotionGoodsByGoods(goodsIds);
kanjiaActivityGoodsService.deleteByGoodsIds(goodsIds);
}
public Map<String, Object> wrapperPromotionMapList(List<PromotionGoods> promotionGoodsList) {
Map<String, Object> promotionMap = new HashMap<>();
for (PromotionGoods promotionGoods : promotionGoodsList) {
String esPromotionKey = promotionGoods.getPromotionType() + "-" + promotionGoods.getPromotionId();
switch (PromotionTypeEnum.valueOf(promotionGoods.getPromotionType())) {
@ -120,13 +131,6 @@ public class PromotionServiceImpl implements PromotionService {
return promotionMap;
}
@Override
public void removeByGoodsIds(String goodsIdsJsonStr) {
List<String> goodsIds = JSONUtil.toList(goodsIdsJsonStr, String.class);
promotionGoodsService.deletePromotionGoodsByGoods(goodsIds);
kanjiaActivityGoodsService.deleteByGoodsIds(goodsIds);
}
private void getGoodsCurrentSeckill(String esPromotionKey, PromotionGoods promotionGoods, Map<String, Object> promotionMap) {
Seckill seckill = seckillService.getById(promotionGoods.getPromotionId());
SeckillSearchParams searchParams = new SeckillSearchParams();

View File

@ -4,7 +4,6 @@ import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.text.CharSequenceUtil;
import cn.hutool.core.thread.ThreadUtil;
import cn.hutool.core.util.ArrayUtil;
import cn.hutool.core.util.ReflectUtil;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
@ -19,10 +18,7 @@ import cn.lili.common.vo.PageVO;
import cn.lili.elasticsearch.BaseElasticsearchService;
import cn.lili.elasticsearch.EsSuffix;
import cn.lili.elasticsearch.config.ElasticsearchProperties;
import cn.lili.modules.goods.entity.dos.Brand;
import cn.lili.modules.goods.entity.dos.Category;
import cn.lili.modules.goods.entity.dos.GoodsSku;
import cn.lili.modules.goods.entity.dos.StoreGoodsLabel;
import cn.lili.modules.goods.entity.dto.GoodsParamsDTO;
import cn.lili.modules.goods.entity.dto.GoodsSkuDTO;
import cn.lili.modules.goods.entity.enums.GoodsAuthEnum;
@ -33,7 +29,9 @@ import cn.lili.modules.goods.service.GoodsSkuService;
import cn.lili.modules.goods.service.StoreGoodsLabelService;
import cn.lili.modules.promotion.entity.dos.BasePromotions;
import cn.lili.modules.promotion.entity.dos.PromotionGoods;
import cn.lili.modules.promotion.entity.enums.PromotionsScopeTypeEnum;
import cn.lili.modules.promotion.entity.enums.PromotionsStatusEnum;
import cn.lili.modules.promotion.service.PromotionGoodsService;
import cn.lili.modules.promotion.service.PromotionService;
import cn.lili.modules.promotion.tools.PromotionTools;
import cn.lili.modules.search.entity.dos.CustomWords;
@ -101,6 +99,9 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
@Autowired
private PromotionService promotionService;
@Autowired
private PromotionGoodsService promotionGoodsService;
@Autowired
private CustomWordsService customWordsService;
@ -129,6 +130,18 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
@Autowired
private ElasticsearchOperations restTemplate;
/**
* 去除 重复元素
*
* @param list
* @return
*/
public static void removeDuplicate(List<String> list) {
HashSet<String> h = new HashSet<>(list);
list.clear();
list.addAll(h);
}
@Override
public void init() {
//获取索引任务标识
@ -173,19 +186,66 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
for (int i = 1; ; i++) {
List<EsGoodsIndex> esGoodsIndices = new ArrayList<>();
Page<GoodsSkuDTO> skuPage = new Page<>(i, 100);
Page<GoodsSkuDTO> skuPage = new Page<>(i, 2000);
IPage<GoodsSkuDTO> skuIPage = goodsSkuService.getGoodsSkuDTOByPage(skuPage, skuQueryWrapper);
if (skuIPage == null || CollUtil.isEmpty(skuIPage.getRecords())) {
break;
}
List<PromotionGoods> skuValidPromotions = promotionGoodsService.findSkuValidPromotions(skuIPage.getRecords());
List<String> brandIds = new ArrayList<>();
List<String> categoryPaths = new ArrayList<>();
List<String> storeCategoryPaths = new ArrayList<>();
for (GoodsSkuDTO goodsSkuDTO : skuIPage.getRecords()) {
if (CharSequenceUtil.isNotEmpty(goodsSkuDTO.getBrandId())) {
brandIds.add(goodsSkuDTO.getBrandId());
}
if (CharSequenceUtil.isNotEmpty(goodsSkuDTO.getStoreCategoryPath())) {
storeCategoryPaths.addAll(Arrays.asList(goodsSkuDTO.getStoreCategoryPath().split(",")));
}
if (CharSequenceUtil.isNotEmpty((goodsSkuDTO.getCategoryPath()))) {
categoryPaths.addAll(Arrays.asList(goodsSkuDTO.getCategoryPath().split(",")));
}
}
List<Map<String, Object>> brandList = new ArrayList<>();
if (CollUtil.isNotEmpty(brandIds)) {
brandList = this.brandService.getBrandsMapsByCategory(brandIds, "id,name,logo");
}
List<Map<String, Object>> categoryList = new ArrayList<>();
if (CollUtil.isNotEmpty(categoryList)) {
categoryList = this.categoryService.listMapsByIdsOrderByLevel(categoryPaths, "id,name");
}
List<Map<String, Object>> storeCategoryList = new ArrayList<>();
if (CollUtil.isNotEmpty(storeCategoryList)) {
storeCategoryList = this.storeGoodsLabelService.listMapsByStoreIds(storeCategoryPaths, "id,label_name");
}
for (GoodsSkuDTO goodsSku : skuIPage.getRecords()) {
int skuSource = 100;
EsGoodsIndex esGoodsIndex = wrapperEsGoodsIndex(goodsSku);
EsGoodsIndex esGoodsIndex = wrapperEsGoodsIndex(goodsSku, brandList, categoryList, storeCategoryList);
long count = esGoodsIndices.stream().filter(j -> j.getGoodsId().equals(esGoodsIndex.getGoodsId())).count();
if (count >= 1) {
skuSource -= count;
}
esGoodsIndex.setSkuSource(skuSource);
//设置促销信息
List<PromotionGoods> promotionGoods = skuValidPromotions.stream()
.filter(j ->
(CharSequenceUtil.isNotEmpty(j.getSkuId()) && j.getSkuId().equals(goodsSku.getId())) ||
j.getScopeType().equals(PromotionsScopeTypeEnum.ALL.name()) ||
(j.getScopeType().equals(PromotionsScopeTypeEnum.PORTION_GOODS_CATEGORY.name()) && j.getScopeId().contains(goodsSku.getCategoryPath())))
.collect(Collectors.toList());
if (CollUtil.isNotEmpty(promotionGoods)) {
esGoodsIndex.setPromotionMapJson(JSONUtil.toJsonStr(promotionService.wrapperPromotionMapList(promotionGoods)));
}
esGoodsIndices.add(esGoodsIndex);
//库存锁是在redis做的所以生成索引同时更新一下redis中的库存数量
cache.put(GoodsSkuService.getStockCacheKey(goodsSku.getId()), goodsSku.getQuantity());
@ -273,7 +333,6 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
goodsIndexRepository.save(goods);
}
/**
* 商品分词
*
@ -309,18 +368,6 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
}
}
/**
* 去除 重复元素
*
* @param list
* @return
*/
public static void removeDuplicate(List<String> list) {
HashSet<String> h = new HashSet<>(list);
list.clear();
list.addAll(h);
}
/**
* 更新商品索引的的部分属性只填写更新的字段不需要更新的字段不要填写
*
@ -853,7 +900,7 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
return elasticsearchProperties.getIndexPrefix() + "_" + EsSuffix.GOODS_INDEX_NAME;
}
private EsGoodsIndex wrapperEsGoodsIndex(GoodsSkuDTO goodsSku) {
private EsGoodsIndex wrapperEsGoodsIndex(GoodsSkuDTO goodsSku, List<Map<String, Object>> brandList, List<Map<String, Object>> categoryList, List<Map<String, Object>> storeCategoryList) {
EsGoodsIndex index = new EsGoodsIndex(goodsSku);
//商品参数索引
@ -862,28 +909,27 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
index = new EsGoodsIndex(goodsSku, goodsParamDTOS);
}
//商品分类索引
if (goodsSku.getCategoryPath() != null) {
List<Category> categories = categoryService.listByIdsOrderByLevel(Arrays.asList(goodsSku.getCategoryPath().split(",")));
if (!categories.isEmpty()) {
index.setCategoryNamePath(ArrayUtil.join(categories.stream().map(Category::getName).toArray(), ","));
}
if (CollUtil.isNotEmpty(categoryList) && CharSequenceUtil.isNotEmpty(goodsSku.getCategoryPath())) {
StringBuilder categoryNamePath = new StringBuilder();
categoryList.stream().filter(o -> goodsSku.getCategoryPath().contains(o.get("id").toString())).forEach(p -> categoryNamePath.append(p.get("name")).append(","));
categoryNamePath.deleteCharAt(categoryNamePath.length() - 1);
index.setCategoryNamePath(categoryNamePath.toString());
}
//商品品牌索引
Brand brand = brandService.getById(goodsSku.getBrandId());
if (brand != null) {
index.setBrandName(brand.getName());
index.setBrandUrl(brand.getLogo());
}
//店铺分类索引
if (goodsSku.getStoreCategoryPath() != null && CharSequenceUtil.isNotEmpty(goodsSku.getStoreCategoryPath())) {
List<StoreGoodsLabel> storeGoodsLabels = storeGoodsLabelService.listByStoreIds(Arrays.asList(goodsSku.getStoreCategoryPath().split(",")));
if (!storeGoodsLabels.isEmpty()) {
index.setStoreCategoryNamePath(ArrayUtil.join(storeGoodsLabels.stream().map(StoreGoodsLabel::getLabelName).toArray(), ","));
if (CollUtil.isNotEmpty(brandList) && CharSequenceUtil.isNotEmpty(goodsSku.getBrandId())) {
Optional<Map<String, Object>> brandInfo = brandList.stream().filter(p -> p.get("id").equals(goodsSku.getBrandId())).findFirst();
if (brandInfo.isPresent()) {
index.setBrandName(brandInfo.get().get("name").toString());
index.setBrandUrl(brandInfo.get().get("logo").toString());
}
}
//促销索引
Map<String, Object> goodsCurrentPromotionMap = promotionService.getGoodsSkuPromotionMap(index.getStoreId(), index.getId());
index.setPromotionMapJson(JSONUtil.toJsonStr(goodsCurrentPromotionMap));
//店铺分类索引
if (CollUtil.isNotEmpty(storeCategoryList) && CharSequenceUtil.isNotEmpty(goodsSku.getStoreCategoryPath())) {
StringBuilder storeCategoryNamePath = new StringBuilder();
storeCategoryList.stream().filter(o -> goodsSku.getStoreCategoryPath().contains(o.get("id").toString())).forEach(p -> storeCategoryNamePath.append(p.get("label_name").toString()).append(","));
storeCategoryNamePath.deleteCharAt(storeCategoryNamePath.length() - 1);
index.setStoreCategoryNamePath(storeCategoryNamePath.toString());
}
return index;
}
}

View File

@ -87,14 +87,6 @@ public class EsGoodsSearchServiceImpl implements EsGoodsSearchService {
NativeSearchQuery searchQuery = searchQueryBuilder.build();
log.debug("searchGoods DSL:{}", searchQuery.getQuery());
SearchHits<EsGoodsIndex> search = restTemplate.search(searchQuery, EsGoodsIndex.class);
// for (int i = 0; i < search.getSearchHits().size() ; i++){
// if (search.getSearchHits().get(i).getContent().getSmall().contains("fuluapiossproductnew.oss-cn-hangzhou.aliyuncs.com")){
// search.getSearchHits().get(i).getContent().setSmall(search.getSearchHits().get(i).getContent().getSmall().replace("?x-oss-process=style/200X200", ""));
// }
// if (search.getSearchHits().get(i).getContent().getThumbnail().contains("fuluapiossproductnew.oss-cn-hangzhou.aliyuncs.com")){
// search.getSearchHits().get(i).getContent().setThumbnail(search.getSearchHits().get(i).getContent().getThumbnail().replace("?x-oss-process=style/400X400", ""));
// }
// }
return SearchHitSupport.searchPageFor(search, searchQuery.getPageable());
}
@ -575,9 +567,9 @@ public class EsGoodsSearchServiceImpl implements EsGoodsSearchService {
filterFunctionBuilders.add(skuNoBuilder);
// 修改分数算法为无数字最大分数越高
FieldValueFactorFunctionBuilder buyCountScore = ScoreFunctionBuilders.fieldValueFactorFunction("buyCount").modifier(FieldValueFactorFunction.Modifier.NONE).setWeight(3);
FunctionScoreQueryBuilder.FilterFunctionBuilder buyCountBuilder = new FunctionScoreQueryBuilder.FilterFunctionBuilder(buyCountScore);
filterFunctionBuilders.add(buyCountBuilder);
// FieldValueFactorFunctionBuilder buyCountScore = ScoreFunctionBuilders.fieldValueFactorFunction("buyCount").modifier(FieldValueFactorFunction.Modifier.NONE).setWeight(10);
// FunctionScoreQueryBuilder.FilterFunctionBuilder buyCountBuilder = new FunctionScoreQueryBuilder.FilterFunctionBuilder(buyCountScore);
// filterFunctionBuilders.add(buyCountBuilder);
return filterFunctionBuilders;
}