!212 优化商品索引的促销信息生成。优化批量生成分词

Merge pull request !212 from OceansDeep/feature/pg
2022-07-18 11:12:12 +00:00 · 2022-07-18 11:12:12 +00:00 · 04f7d22e3f
commit 04f7d22e3f
parent c6596fff1a 950274f6be
10 changed files with 136 additions and 56 deletions
--- a/config/application.yml
+++ b/config/application.yml
@ -29,7 +29,7 @@ spring:
    type: redis
  # Redis
  redis:
-    host: 192.168.0.116
+    host: 127.0.0.1
    port: 6379
    password: lilishop
    lettuce:
@ -60,7 +60,7 @@ spring:
      default-datasource:
        type: com.alibaba.druid.pool.DruidDataSource
        driverClassName: com.mysql.cj.jdbc.Driver
-        url: jdbc:mysql://192.168.0.116:3306/lilishop?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
+        url: jdbc:mysql://127.0.0.1:3306/lilishop?useUnicode=true&characterEncoding=utf-8&useSSL=false&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
        username: root
        password: lilishop
        maxActive: 50
@ -119,6 +119,7 @@ ignored:
    - /store/passport/login/refresh/**
    - /common/common/slider/**
    - /common/common/sms/**
+    - /common/common/site
    - /buyer/payment/cashier/**
    - /buyer/other/pageData/**
    - /buyer/other/article/**
@ -268,7 +269,7 @@ lili:
  data:
    elasticsearch:
      cluster-name: elasticsearch
-      cluster-nodes: 192.168.0.116:9200
+      cluster-nodes: 127.0.0.1:9200
      index:
        number-of-replicas: 0
        number-of-shards: 3
@ -279,7 +280,7 @@ lili:
    #        password: LiLiShopES

    logstash:
-      server: 192.168.0.116:4560
+      server: 127.0.0.1:4560
    rocketmq:
      promotion-topic: lili_promotion_topic
      promotion-group: lili_promotion_group
@ -300,7 +301,7 @@ lili:
      after-sale-topic: lili_after_sale_topic
      after-sale-group: lili_after_sale_group
 rocketmq:
-  name-server: 192.168.0.116:9876
+  name-server: 127.0.0.1:9876
  isVIPChannel: false
  producer:
    group: lili_group
@ -309,7 +310,7 @@ rocketmq:
 xxl:
  job:
    admin:
-      addresses: http://192.168.0.116:9001/xxl-job-admin
+      addresses: http://127.0.0.1:9001/xxl-job-admin
    executor:
      appname: xxl-job-executor-lilishop
      address:
--- a/consumer/src/main/java/cn/lili/listener/GoodsMessageListener.java
+++ b/consumer/src/main/java/cn/lili/listener/GoodsMessageListener.java
@ -275,8 +275,6 @@ public class GoodsMessageListener implements RocketMQListener<MessageExt> {
                this.goodsIndexService.deleteEsGoodsPromotionByPromotionKey(skuIds, esPromotionKey);
                this.goodsIndexService.updateEsGoodsIndexPromotions(skuIds, promotions, esPromotionKey);
            } else if (PromotionsScopeTypeEnum.ALL.name().equals(promotions.getScopeType())) {
-                // 更新商品索引促销信息（删除原索引中相关的促销信息，更新索引中促销信息）
-                this.goodsIndexService.deleteEsGoodsPromotionByPromotionKey(esPromotionKey);
                this.goodsIndexService.updateEsGoodsIndexAllByList(promotions, esPromotionKey);
            }
        } catch (Exception e) {
--- a/consumer/src/main/resources/application.yml
+++ b/consumer/src/main/resources/application.yml
@ -67,7 +67,7 @@ spring:
      default-datasource:
        type: com.alibaba.druid.pool.DruidDataSource
        driverClassName: com.mysql.cj.jdbc.Driver
-        url: jdbc:mysql://127.0.0.1:3306/lilishop?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
+        url: jdbc:mysql://127.0.0.1:3306/lilishop?useUnicode=true&characterEncoding=utf-8&useSSL=false&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
        username: root
        password: lilishop
        maxActive: 20
--- a/framework/src/main/java/cn/lili/modules/search/entity/dos/CustomWords.java
+++ b/framework/src/main/java/cn/lili/modules/search/entity/dos/CustomWords.java
@ -4,8 +4,10 @@ import cn.lili.mybatis.BaseEntity;
 import com.baomidou.mybatisplus.annotation.TableName;
 import io.swagger.annotations.ApiModel;
 import io.swagger.annotations.ApiModelProperty;
+import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
 import org.hibernate.validator.constraints.Length;

 import javax.validation.constraints.NotEmpty;
@ -20,6 +22,8 @@ import javax.validation.constraints.NotEmpty;
@TableName("li_custom_words")
@ApiModel(value = "自定义分词")
@EqualsAndHashCode(callSuper = true)
+@NoArgsConstructor
+@AllArgsConstructor
 public class CustomWords extends BaseEntity {

    private static final long serialVersionUID = 650889506808657977L;
--- a/framework/src/main/java/cn/lili/modules/search/mapper/CustomWordsMapper.java
+++ b/framework/src/main/java/cn/lili/modules/search/mapper/CustomWordsMapper.java
@ -1,6 +1,7 @@
 package cn.lili.modules.search.mapper;

 import cn.lili.modules.search.entity.dos.CustomWords;
+import cn.lili.mybatis.mybatisplus.SpiceBaseMapper;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;

 /**
@ -9,5 +10,5 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 * @author paulG
 * @since 2020/10/15
 **/
-public interface CustomWordsMapper extends BaseMapper<CustomWords> {
+public interface CustomWordsMapper extends BaseMapper<CustomWords>, SpiceBaseMapper<CustomWords> {
 }
--- a/framework/src/main/java/cn/lili/modules/search/service/CustomWordsService.java
+++ b/framework/src/main/java/cn/lili/modules/search/service/CustomWordsService.java
@ -6,6 +6,8 @@ import cn.lili.modules.search.entity.vo.CustomWordsVO;
 import com.baomidou.mybatisplus.core.metadata.IPage;
 import com.baomidou.mybatisplus.extension.service.IService;

+import java.util.List;
+
 /**
 * 自定义分词业务层
 *
@ -52,6 +54,16 @@ public interface CustomWordsService extends IService<CustomWords> {
     */
    boolean deleteCustomWords(String id);

+    /**
+     * 根据名字批量删除
+     *
+     * @param names 名称列表
+     * @return 是否删除成功
+     */
+    boolean deleteBathByName(List<String> names);
+
+    long insertBatchCustomWords(List<CustomWords> customWordsList);
+
    /**
     * 分页查询自定义分词
     *
--- a/framework/src/main/java/cn/lili/modules/search/serviceimpl/CustomWordsServiceImpl.java
+++ b/framework/src/main/java/cn/lili/modules/search/serviceimpl/CustomWordsServiceImpl.java
@ -91,6 +91,18 @@ public class CustomWordsServiceImpl extends ServiceImpl<CustomWordsMapper, Custo
        return this.removeById(id);
    }

+    @Override
+    public boolean deleteBathByName(List<String> names) {
+        LambdaQueryWrapper<CustomWords> queryWrapper = new LambdaQueryWrapper<>();
+        queryWrapper.in(CustomWords::getName, names);
+        return this.remove(queryWrapper);
+    }
+
+    @Override
+    public long insertBatchCustomWords(List<CustomWords> customWordsList) {
+        return this.baseMapper.insertBatchSomeColumn(customWordsList);
+    }
+
    /**
     * 修改自定义分词
     *
--- a/framework/src/main/java/cn/lili/modules/search/serviceimpl/EsGoodsIndexServiceImpl.java
+++ b/framework/src/main/java/cn/lili/modules/search/serviceimpl/EsGoodsIndexServiceImpl.java
@ -15,6 +15,7 @@ import cn.lili.common.enums.ResultCode;
 import cn.lili.common.exception.RetryException;
 import cn.lili.common.exception.ServiceException;
 import cn.lili.common.properties.RocketmqCustomProperties;
+import cn.lili.common.vo.PageVO;
 import cn.lili.elasticsearch.BaseElasticsearchService;
 import cn.lili.elasticsearch.EsSuffix;
 import cn.lili.elasticsearch.config.ElasticsearchProperties;
@ -35,10 +36,10 @@ import cn.lili.modules.promotion.entity.dos.PromotionGoods;
 import cn.lili.modules.promotion.entity.enums.PromotionsStatusEnum;
 import cn.lili.modules.promotion.service.PromotionService;
 import cn.lili.modules.promotion.tools.PromotionTools;
+import cn.lili.modules.search.entity.dos.CustomWords;
 import cn.lili.modules.search.entity.dos.EsGoodsAttribute;
 import cn.lili.modules.search.entity.dos.EsGoodsIndex;
 import cn.lili.modules.search.entity.dto.EsGoodsSearchDTO;
-import cn.lili.modules.search.entity.vo.CustomWordsVO;
 import cn.lili.modules.search.repository.EsGoodsIndexRepository;
 import cn.lili.modules.search.service.CustomWordsService;
 import cn.lili.modules.search.service.EsGoodsIndexService;
@ -50,7 +51,6 @@ import com.baomidou.mybatisplus.core.metadata.IPage;
 import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.rocketmq.spring.core.RocketMQTemplate;
-import org.assertj.core.util.IterableUtil;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.update.UpdateRequest;
@ -65,8 +65,8 @@ import org.elasticsearch.index.reindex.DeleteByQueryRequest;
 import org.elasticsearch.index.reindex.UpdateByQueryRequest;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
-import org.mybatis.spring.MyBatisSystemException;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.domain.PageRequest;
 import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
 import org.springframework.data.elasticsearch.core.SearchHit;
 import org.springframework.data.elasticsearch.core.SearchPage;
@ -287,17 +287,28 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
            AnalyzeRequest analyzeRequest = AnalyzeRequest.withIndexAnalyzer(getIndexName(), "ik_max_word", goods.getGoodsName());
            AnalyzeResponse analyze = client.indices().analyze(analyzeRequest, RequestOptions.DEFAULT);
            List<AnalyzeResponse.AnalyzeToken> tokens = analyze.getTokens();
-
+            List<CustomWords> customWordsList = new ArrayList<>();
+            List<String> keywordsList = new ArrayList<>();
            if (goods.getAttrList() != null && !goods.getAttrList().isEmpty()) {
                //保存分词
                for (EsGoodsAttribute esGoodsAttribute : goods.getAttrList()) {
-                    wordsToDb(esGoodsAttribute.getValue());
+                    if (keywordsList.stream().noneMatch(i -> i.toLowerCase(Locale.ROOT).equals(esGoodsAttribute.getValue().toLowerCase(Locale.ROOT)))) {
+                        keywordsList.add(esGoodsAttribute.getValue());
+                        customWordsList.add(new CustomWords(esGoodsAttribute.getValue(), 0));
+                    }
                }
            }
            //分析词条
            for (AnalyzeResponse.AnalyzeToken token : tokens) {
+                if (keywordsList.stream().noneMatch(i -> i.toLowerCase(Locale.ROOT).equals(token.getTerm().toLowerCase(Locale.ROOT)))) {
+                    keywordsList.add(token.getTerm());
+                    customWordsList.add(new CustomWords(token.getTerm(), 0));
+                }
                //保存词条进入数据库
-                wordsToDb(token.getTerm());
+            }
+            if (CollUtil.isNotEmpty(customWordsList)) {
+                customWordsService.deleteBathByName(keywordsList);
+                customWordsService.insertBatchCustomWords(customWordsList);
            }
        } catch (IOException e) {
            log.info(goods + "分词错误", e);
@ -537,25 +548,35 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
    @Override
    public void updateEsGoodsIndexAllByList(BasePromotions promotion, String key) {
        ThreadUtil.execAsync(() -> {
-            List<EsGoodsIndex> goodsIndices = new ArrayList<>();
-            //如果storeId不为空，则表示是店铺活动
-            if (promotion.getStoreId() != null && !promotion.getStoreId().equals(PromotionTools.PLATFORM_ID)) {
-                EsGoodsSearchDTO searchDTO = new EsGoodsSearchDTO();
-                searchDTO.setStoreId(promotion.getStoreId());
-                //查询出店铺商品
-                SearchPage<EsGoodsIndex> esGoodsIndices = goodsSearchService.searchGoods(searchDTO, null);
-                for (SearchHit<EsGoodsIndex> searchHit : esGoodsIndices.getContent()) {
-                    goodsIndices.add(searchHit.getContent());
+            for (int i = 1; ; i++) {
+                List<String> skuIds;
+                //如果storeId不为空，则表示是店铺活动
+                if (promotion.getStoreId() != null && !promotion.getStoreId().equals(PromotionTools.PLATFORM_ID)) {
+                    PageVO pageVO = new PageVO();
+                    pageVO.setPageNumber(i);
+                    pageVO.setPageSize(1000);
+                    EsGoodsSearchDTO searchDTO = new EsGoodsSearchDTO();
+                    searchDTO.setStoreId(promotion.getStoreId());
+                    //查询出店铺商品
+                    SearchPage<EsGoodsIndex> esGoodsIndices = goodsSearchService.searchGoods(searchDTO, pageVO);
+
+                    skuIds = esGoodsIndices.isEmpty() ? new ArrayList<>() : esGoodsIndices.getContent().stream().map(SearchHit::getId).collect(Collectors.toList());
+                } else {
+                    //否则是平台活动
+                    org.springframework.data.domain.Page<EsGoodsIndex> all = goodsIndexRepository.findAll(PageRequest.of(i, 1000));
+
+                    //查询出全部商品
+                    skuIds = all.isEmpty() ? new ArrayList<>() : all.toList().stream().map(EsGoodsIndex::getId).collect(Collectors.toList());
                }
-            } else {
-                //否则是平台活动
-                Iterable<EsGoodsIndex> all = goodsIndexRepository.findAll();
-                //查询出全部商品
-                goodsIndices = new ArrayList<>(IterableUtil.toCollection(all));
+                if (skuIds.isEmpty()) {
+                    break;
+                }
+                this.deleteEsGoodsPromotionByPromotionKey(skuIds, key);
+                this.updateEsGoodsIndexPromotions(skuIds, promotion, key);
            }
-            List<String> skuIds = goodsIndices.stream().map(EsGoodsIndex::getId).collect(Collectors.toList());
-            this.updateEsGoodsIndexPromotions(skuIds, promotion, key);
+
        });
+
    }

    @Override
@ -730,9 +751,8 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
        } else {
            promotionMap = goodsIndex.getOriginPromotionMap();
        }
-
-        log.info("ES修改商品活动索引-原商品索引信息:{}", goodsIndex);
-        log.info("ES修改商品活动索引-原商品索引活动信息:{}", promotionMap);
+//        log.info("ES修改商品活动索引-原商品索引信息:{}", goodsIndex);
+//        log.info("ES修改商品活动索引-原商品索引活动信息:{}", promotionMap);
        //如果活动已结束
        if (promotion.getPromotionStatus().equals(PromotionsStatusEnum.END.name()) || promotion.getPromotionStatus().equals(PromotionsStatusEnum.CLOSE.name())) {//如果存在活动
            //删除活动
@ -740,7 +760,7 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
        } else {
            promotionMap.put(key, promotion);
        }
-        log.info("ES修改商品活动索引-过滤后商品索引活动信息:{}", promotionMap);
+//        log.info("ES修改商品活动索引-过滤后商品索引活动信息:{}", promotionMap);
        return this.getGoodsIndexPromotionUpdateRequest(goodsIndex.getId(), promotionMap);
    }

@ -759,7 +779,6 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
        Map<String, Object> params = new HashMap<>();
        params.put("promotionMap", JSONUtil.toJsonStr(promotionMap));
        Script script = new Script(ScriptType.INLINE, "painless", "ctx._source.promotionMapJson=params.promotionMap;", params);
-        log.info("执行脚本内容：{}", script);
        updateRequest.script(script);
        return updateRequest;
    }
@ -812,25 +831,6 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
        }
    }

-    /**
-     * 将商品关键字入库
-     *
-     * @param words 商品关键字
-     */
-    private void wordsToDb(String words) {
-        if (CharSequenceUtil.isEmpty(words)) {
-            return;
-        }
-        try {
-            //是否有重复
-            customWordsService.addCustomWords(new CustomWordsVO(words));
-        } catch (MyBatisSystemException me) {
-            log.error(words + "关键字已存在！", me);
-        } catch (Exception e) {
-            log.error("关键字入库异常！", e);
-        }
-    }
-
    private String getIndexName() {
        //索引名称拼接
        return elasticsearchProperties.getIndexPrefix() + "_" + EsSuffix.GOODS_INDEX_NAME;
--- a/framework/src/main/java/cn/lili/mybatis/mybatisplus/SpiceBaseMapper.java
+++ b/framework/src/main/java/cn/lili/mybatis/mybatisplus/SpiceBaseMapper.java
@ -0,0 +1,19 @@
+package cn.lili.mybatis.mybatisplus;
+
+import java.util.List;
+
+/**
+ * @author paulG
+ * @since 2022/7/18
+ **/
+public interface SpiceBaseMapper<T> {
+
+    /**
+     * 批量插入
+     * {@link com.baomidou.mybatisplus.extension.injector.methods.InsertBatchSomeColumn}
+     *
+     * @param entityList 要插入的数据
+     * @return 成功插入的数据条数
+     */
+    long insertBatchSomeColumn(List<T> entityList);
+}
--- a/framework/src/main/java/cn/lili/mybatis/mybatisplus/SpiceSqlInjector.java
+++ b/framework/src/main/java/cn/lili/mybatis/mybatisplus/SpiceSqlInjector.java
@ -0,0 +1,33 @@
+package cn.lili.mybatis.mybatisplus;
+
+import com.baomidou.mybatisplus.core.injector.AbstractMethod;
+import com.baomidou.mybatisplus.core.injector.DefaultSqlInjector;
+import com.baomidou.mybatisplus.core.metadata.TableInfo;
+import com.baomidou.mybatisplus.extension.injector.methods.InsertBatchSomeColumn;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+
+/**
+ * @author paulG
+ * @since 2022/7/18
+ **/
+@Component
+public class SpiceSqlInjector extends DefaultSqlInjector {
+
+    /**
+     * 如果只需增加方法，保留mybatis plus自带方法，
+     * 可以先获取super.getMethodList()，再添加add
+     */
+    @Override
+    public List<AbstractMethod> getMethodList(Class<?> mapperClass, TableInfo tableInfo) {
+        // 注意：此SQL注入器继承了DefaultSqlInjector(默认注入器)，调用了DefaultSqlInjector的getMethodList方法，保留了mybatis-plus的自带方法
+        List<AbstractMethod> methodList = super.getMethodList(mapperClass, tableInfo);
+        // 注入InsertBatchSomeColumn
+        // 在!t.isLogicDelete()表示不要逻辑删除字段，!"update_time".equals(t.getColumn())表示不要字段名为 update_time 的字段,不对进行操作
+        // methodList.add(new InsertBatchSomeColumn(t -> !t.isLogicDelete() && !"update_time".equals(t.getColumn())));
+        // 要逻辑删除 t.isLogicDelete() 默认不要
+        methodList.add(new InsertBatchSomeColumn(t -> !t.isLogicDelete()));
+        return methodList;
+    }
+}