From 950274f6bee20e092f4d7168a36d812f4e178ab9 Mon Sep 17 00:00:00 2001 From: paulGao Date: Mon, 18 Jul 2022 19:06:17 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=95=86=E5=93=81=E7=B4=A2?= =?UTF-8?q?=E5=BC=95=E7=9A=84=E4=BF=83=E9=94=80=E4=BF=A1=E6=81=AF=E7=94=9F?= =?UTF-8?q?=E6=88=90=E3=80=82=E4=BC=98=E5=8C=96=E6=89=B9=E9=87=8F=E7=94=9F?= =?UTF-8?q?=E6=88=90=E5=88=86=E8=AF=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/application.yml | 13 +-- .../lili/listener/GoodsMessageListener.java | 2 - consumer/src/main/resources/application.yml | 2 +- .../search/entity/dos/CustomWords.java | 4 + .../search/mapper/CustomWordsMapper.java | 3 +- .../search/service/CustomWordsService.java | 12 +++ .../serviceimpl/CustomWordsServiceImpl.java | 12 +++ .../serviceimpl/EsGoodsIndexServiceImpl.java | 92 +++++++++---------- .../mybatis/mybatisplus/SpiceBaseMapper.java | 19 ++++ .../mybatis/mybatisplus/SpiceSqlInjector.java | 33 +++++++ 10 files changed, 136 insertions(+), 56 deletions(-) create mode 100644 framework/src/main/java/cn/lili/mybatis/mybatisplus/SpiceBaseMapper.java create mode 100644 framework/src/main/java/cn/lili/mybatis/mybatisplus/SpiceSqlInjector.java diff --git a/config/application.yml b/config/application.yml index 703b96ae..201f837f 100644 --- a/config/application.yml +++ b/config/application.yml @@ -29,7 +29,7 @@ spring: type: redis # Redis redis: - host: 192.168.0.116 + host: 127.0.0.1 port: 6379 password: lilishop lettuce: @@ -60,7 +60,7 @@ spring: default-datasource: type: com.alibaba.druid.pool.DruidDataSource driverClassName: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://192.168.0.116:3306/lilishop?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai + url: jdbc:mysql://127.0.0.1:3306/lilishop?useUnicode=true&characterEncoding=utf-8&useSSL=false&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai username: root password: lilishop maxActive: 50 @@ -119,6 +119,7 @@ ignored: - /store/passport/login/refresh/** - /common/common/slider/** - /common/common/sms/** + - /common/common/site - /buyer/payment/cashier/** - /buyer/other/pageData/** - /buyer/other/article/** @@ -268,7 +269,7 @@ lili: data: elasticsearch: cluster-name: elasticsearch - cluster-nodes: 192.168.0.116:9200 + cluster-nodes: 127.0.0.1:9200 index: number-of-replicas: 0 number-of-shards: 3 @@ -279,7 +280,7 @@ lili: # password: LiLiShopES logstash: - server: 192.168.0.116:4560 + server: 127.0.0.1:4560 rocketmq: promotion-topic: lili_promotion_topic promotion-group: lili_promotion_group @@ -300,7 +301,7 @@ lili: after-sale-topic: lili_after_sale_topic after-sale-group: lili_after_sale_group rocketmq: - name-server: 192.168.0.116:9876 + name-server: 127.0.0.1:9876 isVIPChannel: false producer: group: lili_group @@ -309,7 +310,7 @@ rocketmq: xxl: job: admin: - addresses: http://192.168.0.116:9001/xxl-job-admin + addresses: http://127.0.0.1:9001/xxl-job-admin executor: appname: xxl-job-executor-lilishop address: diff --git a/consumer/src/main/java/cn/lili/listener/GoodsMessageListener.java b/consumer/src/main/java/cn/lili/listener/GoodsMessageListener.java index 429a3950..367c7425 100644 --- a/consumer/src/main/java/cn/lili/listener/GoodsMessageListener.java +++ b/consumer/src/main/java/cn/lili/listener/GoodsMessageListener.java @@ -275,8 +275,6 @@ public class GoodsMessageListener implements RocketMQListener { this.goodsIndexService.deleteEsGoodsPromotionByPromotionKey(skuIds, esPromotionKey); this.goodsIndexService.updateEsGoodsIndexPromotions(skuIds, promotions, esPromotionKey); } else if (PromotionsScopeTypeEnum.ALL.name().equals(promotions.getScopeType())) { - // 更新商品索引促销信息（删除原索引中相关的促销信息，更新索引中促销信息） - this.goodsIndexService.deleteEsGoodsPromotionByPromotionKey(esPromotionKey); this.goodsIndexService.updateEsGoodsIndexAllByList(promotions, esPromotionKey); } } catch (Exception e) { diff --git a/consumer/src/main/resources/application.yml b/consumer/src/main/resources/application.yml index 8304cf32..ae49a9b8 100644 --- a/consumer/src/main/resources/application.yml +++ b/consumer/src/main/resources/application.yml @@ -67,7 +67,7 @@ spring: default-datasource: type: com.alibaba.druid.pool.DruidDataSource driverClassName: com.mysql.cj.jdbc.Driver - url: jdbc:mysql://127.0.0.1:3306/lilishop?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai + url: jdbc:mysql://127.0.0.1:3306/lilishop?useUnicode=true&characterEncoding=utf-8&useSSL=false&rewriteBatchedStatements=true&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai username: root password: lilishop maxActive: 20 diff --git a/framework/src/main/java/cn/lili/modules/search/entity/dos/CustomWords.java b/framework/src/main/java/cn/lili/modules/search/entity/dos/CustomWords.java index 745e8ab9..4b53e75f 100644 --- a/framework/src/main/java/cn/lili/modules/search/entity/dos/CustomWords.java +++ b/framework/src/main/java/cn/lili/modules/search/entity/dos/CustomWords.java @@ -4,8 +4,10 @@ import cn.lili.mybatis.BaseEntity; import com.baomidou.mybatisplus.annotation.TableName; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; import org.hibernate.validator.constraints.Length; import javax.validation.constraints.NotEmpty; @@ -20,6 +22,8 @@ import javax.validation.constraints.NotEmpty; @TableName("li_custom_words") @ApiModel(value = "自定义分词") @EqualsAndHashCode(callSuper = true) +@NoArgsConstructor +@AllArgsConstructor public class CustomWords extends BaseEntity { private static final long serialVersionUID = 650889506808657977L; diff --git a/framework/src/main/java/cn/lili/modules/search/mapper/CustomWordsMapper.java b/framework/src/main/java/cn/lili/modules/search/mapper/CustomWordsMapper.java index 359cd238..858eda75 100644 --- a/framework/src/main/java/cn/lili/modules/search/mapper/CustomWordsMapper.java +++ b/framework/src/main/java/cn/lili/modules/search/mapper/CustomWordsMapper.java @@ -1,6 +1,7 @@ package cn.lili.modules.search.mapper; import cn.lili.modules.search.entity.dos.CustomWords; +import cn.lili.mybatis.mybatisplus.SpiceBaseMapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; /** @@ -9,5 +10,5 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper; * @author paulG * @since 2020/10/15 **/ -public interface CustomWordsMapper extends BaseMapper { +public interface CustomWordsMapper extends BaseMapper, SpiceBaseMapper { } diff --git a/framework/src/main/java/cn/lili/modules/search/service/CustomWordsService.java b/framework/src/main/java/cn/lili/modules/search/service/CustomWordsService.java index 5193cdf1..72621949 100644 --- a/framework/src/main/java/cn/lili/modules/search/service/CustomWordsService.java +++ b/framework/src/main/java/cn/lili/modules/search/service/CustomWordsService.java @@ -6,6 +6,8 @@ import cn.lili.modules.search.entity.vo.CustomWordsVO; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.service.IService; +import java.util.List; + /** * 自定义分词业务层 * @@ -52,6 +54,16 @@ public interface CustomWordsService extends IService { */ boolean deleteCustomWords(String id); + /** + * 根据名字批量删除 + * + * @param names 名称列表 + * @return 是否删除成功 + */ + boolean deleteBathByName(List names); + + long insertBatchCustomWords(List customWordsList); + /** * 分页查询自定义分词 * diff --git a/framework/src/main/java/cn/lili/modules/search/serviceimpl/CustomWordsServiceImpl.java b/framework/src/main/java/cn/lili/modules/search/serviceimpl/CustomWordsServiceImpl.java index 1454c3b0..405050fa 100644 --- a/framework/src/main/java/cn/lili/modules/search/serviceimpl/CustomWordsServiceImpl.java +++ b/framework/src/main/java/cn/lili/modules/search/serviceimpl/CustomWordsServiceImpl.java @@ -91,6 +91,18 @@ public class CustomWordsServiceImpl extends ServiceImpl names) { + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); + queryWrapper.in(CustomWords::getName, names); + return this.remove(queryWrapper); + } + + @Override + public long insertBatchCustomWords(List customWordsList) { + return this.baseMapper.insertBatchSomeColumn(customWordsList); + } + /** * 修改自定义分词 * diff --git a/framework/src/main/java/cn/lili/modules/search/serviceimpl/EsGoodsIndexServiceImpl.java b/framework/src/main/java/cn/lili/modules/search/serviceimpl/EsGoodsIndexServiceImpl.java index f0ec4a56..4695102d 100644 --- a/framework/src/main/java/cn/lili/modules/search/serviceimpl/EsGoodsIndexServiceImpl.java +++ b/framework/src/main/java/cn/lili/modules/search/serviceimpl/EsGoodsIndexServiceImpl.java @@ -15,6 +15,7 @@ import cn.lili.common.enums.ResultCode; import cn.lili.common.exception.RetryException; import cn.lili.common.exception.ServiceException; import cn.lili.common.properties.RocketmqCustomProperties; +import cn.lili.common.vo.PageVO; import cn.lili.elasticsearch.BaseElasticsearchService; import cn.lili.elasticsearch.EsSuffix; import cn.lili.elasticsearch.config.ElasticsearchProperties; @@ -35,10 +36,10 @@ import cn.lili.modules.promotion.entity.dos.PromotionGoods; import cn.lili.modules.promotion.entity.enums.PromotionsStatusEnum; import cn.lili.modules.promotion.service.PromotionService; import cn.lili.modules.promotion.tools.PromotionTools; +import cn.lili.modules.search.entity.dos.CustomWords; import cn.lili.modules.search.entity.dos.EsGoodsAttribute; import cn.lili.modules.search.entity.dos.EsGoodsIndex; import cn.lili.modules.search.entity.dto.EsGoodsSearchDTO; -import cn.lili.modules.search.entity.vo.CustomWordsVO; import cn.lili.modules.search.repository.EsGoodsIndexRepository; import cn.lili.modules.search.service.CustomWordsService; import cn.lili.modules.search.service.EsGoodsIndexService; @@ -50,7 +51,6 @@ import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import lombok.extern.slf4j.Slf4j; import org.apache.rocketmq.spring.core.RocketMQTemplate; -import org.assertj.core.util.IterableUtil; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.update.UpdateRequest; @@ -65,8 +65,8 @@ import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.index.reindex.UpdateByQueryRequest; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; -import org.mybatis.spring.MyBatisSystemException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; import org.springframework.data.elasticsearch.core.ElasticsearchOperations; import org.springframework.data.elasticsearch.core.SearchHit; import org.springframework.data.elasticsearch.core.SearchPage; @@ -287,17 +287,28 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements AnalyzeRequest analyzeRequest = AnalyzeRequest.withIndexAnalyzer(getIndexName(), "ik_max_word", goods.getGoodsName()); AnalyzeResponse analyze = client.indices().analyze(analyzeRequest, RequestOptions.DEFAULT); List tokens = analyze.getTokens(); - + List customWordsList = new ArrayList<>(); + List keywordsList = new ArrayList<>(); if (goods.getAttrList() != null && !goods.getAttrList().isEmpty()) { //保存分词 for (EsGoodsAttribute esGoodsAttribute : goods.getAttrList()) { - wordsToDb(esGoodsAttribute.getValue()); + if (keywordsList.stream().noneMatch(i -> i.toLowerCase(Locale.ROOT).equals(esGoodsAttribute.getValue().toLowerCase(Locale.ROOT)))) { + keywordsList.add(esGoodsAttribute.getValue()); + customWordsList.add(new CustomWords(esGoodsAttribute.getValue(), 0)); + } } } //分析词条 for (AnalyzeResponse.AnalyzeToken token : tokens) { + if (keywordsList.stream().noneMatch(i -> i.toLowerCase(Locale.ROOT).equals(token.getTerm().toLowerCase(Locale.ROOT)))) { + keywordsList.add(token.getTerm()); + customWordsList.add(new CustomWords(token.getTerm(), 0)); + } //保存词条进入数据库 - wordsToDb(token.getTerm()); + } + if (CollUtil.isNotEmpty(customWordsList)) { + customWordsService.deleteBathByName(keywordsList); + customWordsService.insertBatchCustomWords(customWordsList); } } catch (IOException e) { log.info(goods + "分词错误", e); @@ -537,25 +548,35 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements @Override public void updateEsGoodsIndexAllByList(BasePromotions promotion, String key) { ThreadUtil.execAsync(() -> { - List goodsIndices = new ArrayList<>(); - //如果storeId不为空，则表示是店铺活动 - if (promotion.getStoreId() != null && !promotion.getStoreId().equals(PromotionTools.PLATFORM_ID)) { - EsGoodsSearchDTO searchDTO = new EsGoodsSearchDTO(); - searchDTO.setStoreId(promotion.getStoreId()); - //查询出店铺商品 - SearchPage esGoodsIndices = goodsSearchService.searchGoods(searchDTO, null); - for (SearchHit searchHit : esGoodsIndices.getContent()) { - goodsIndices.add(searchHit.getContent()); + for (int i = 1; ; i++) { + List skuIds; + //如果storeId不为空，则表示是店铺活动 + if (promotion.getStoreId() != null && !promotion.getStoreId().equals(PromotionTools.PLATFORM_ID)) { + PageVO pageVO = new PageVO(); + pageVO.setPageNumber(i); + pageVO.setPageSize(1000); + EsGoodsSearchDTO searchDTO = new EsGoodsSearchDTO(); + searchDTO.setStoreId(promotion.getStoreId()); + //查询出店铺商品 + SearchPage esGoodsIndices = goodsSearchService.searchGoods(searchDTO, pageVO); + + skuIds = esGoodsIndices.isEmpty() ? new ArrayList<>() : esGoodsIndices.getContent().stream().map(SearchHit::getId).collect(Collectors.toList()); + } else { + //否则是平台活动 + org.springframework.data.domain.Page all = goodsIndexRepository.findAll(PageRequest.of(i, 1000)); + + //查询出全部商品 + skuIds = all.isEmpty() ? new ArrayList<>() : all.toList().stream().map(EsGoodsIndex::getId).collect(Collectors.toList()); } - } else { - //否则是平台活动 - Iterable all = goodsIndexRepository.findAll(); - //查询出全部商品 - goodsIndices = new ArrayList<>(IterableUtil.toCollection(all)); + if (skuIds.isEmpty()) { + break; + } + this.deleteEsGoodsPromotionByPromotionKey(skuIds, key); + this.updateEsGoodsIndexPromotions(skuIds, promotion, key); } - List skuIds = goodsIndices.stream().map(EsGoodsIndex::getId).collect(Collectors.toList()); - this.updateEsGoodsIndexPromotions(skuIds, promotion, key); + }); + } @Override @@ -730,9 +751,8 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements } else { promotionMap = goodsIndex.getOriginPromotionMap(); } - - log.info("ES修改商品活动索引-原商品索引信息:{}", goodsIndex); - log.info("ES修改商品活动索引-原商品索引活动信息:{}", promotionMap); +// log.info("ES修改商品活动索引-原商品索引信息:{}", goodsIndex); +// log.info("ES修改商品活动索引-原商品索引活动信息:{}", promotionMap); //如果活动已结束 if (promotion.getPromotionStatus().equals(PromotionsStatusEnum.END.name()) || promotion.getPromotionStatus().equals(PromotionsStatusEnum.CLOSE.name())) {//如果存在活动 //删除活动 @@ -740,7 +760,7 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements } else { promotionMap.put(key, promotion); } - log.info("ES修改商品活动索引-过滤后商品索引活动信息:{}", promotionMap); +// log.info("ES修改商品活动索引-过滤后商品索引活动信息:{}", promotionMap); return this.getGoodsIndexPromotionUpdateRequest(goodsIndex.getId(), promotionMap); } @@ -759,7 +779,6 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements Map params = new HashMap<>(); params.put("promotionMap", JSONUtil.toJsonStr(promotionMap)); Script script = new Script(ScriptType.INLINE, "painless", "ctx._source.promotionMapJson=params.promotionMap;", params); - log.info("执行脚本内容：{}", script); updateRequest.script(script); return updateRequest; } @@ -812,25 +831,6 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements } } - /** - * 将商品关键字入库 - * - * @param words 商品关键字 - */ - private void wordsToDb(String words) { - if (CharSequenceUtil.isEmpty(words)) { - return; - } - try { - //是否有重复 - customWordsService.addCustomWords(new CustomWordsVO(words)); - } catch (MyBatisSystemException me) { - log.error(words + "关键字已存在！", me); - } catch (Exception e) { - log.error("关键字入库异常！", e); - } - } - private String getIndexName() { //索引名称拼接 return elasticsearchProperties.getIndexPrefix() + "_" + EsSuffix.GOODS_INDEX_NAME; diff --git a/framework/src/main/java/cn/lili/mybatis/mybatisplus/SpiceBaseMapper.java b/framework/src/main/java/cn/lili/mybatis/mybatisplus/SpiceBaseMapper.java new file mode 100644 index 00000000..c72055b9 --- /dev/null +++ b/framework/src/main/java/cn/lili/mybatis/mybatisplus/SpiceBaseMapper.java @@ -0,0 +1,19 @@ +package cn.lili.mybatis.mybatisplus; + +import java.util.List; + +/** + * @author paulG + * @since 2022/7/18 + **/ +public interface SpiceBaseMapper { + + /** + * 批量插入 + * {@link com.baomidou.mybatisplus.extension.injector.methods.InsertBatchSomeColumn} + * + * @param entityList 要插入的数据 + * @return 成功插入的数据条数 + */ + long insertBatchSomeColumn(List entityList); +} diff --git a/framework/src/main/java/cn/lili/mybatis/mybatisplus/SpiceSqlInjector.java b/framework/src/main/java/cn/lili/mybatis/mybatisplus/SpiceSqlInjector.java new file mode 100644 index 00000000..a7d86ae8 --- /dev/null +++ b/framework/src/main/java/cn/lili/mybatis/mybatisplus/SpiceSqlInjector.java @@ -0,0 +1,33 @@ +package cn.lili.mybatis.mybatisplus; + +import com.baomidou.mybatisplus.core.injector.AbstractMethod; +import com.baomidou.mybatisplus.core.injector.DefaultSqlInjector; +import com.baomidou.mybatisplus.core.metadata.TableInfo; +import com.baomidou.mybatisplus.extension.injector.methods.InsertBatchSomeColumn; +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * @author paulG + * @since 2022/7/18 + **/ +@Component +public class SpiceSqlInjector extends DefaultSqlInjector { + + /** + * 如果只需增加方法，保留mybatis plus自带方法， + * 可以先获取super.getMethodList()，再添加add + */ + @Override + public List getMethodList(Class mapperClass, TableInfo tableInfo) { + // 注意：此SQL注入器继承了DefaultSqlInjector(默认注入器)，调用了DefaultSqlInjector的getMethodList方法，保留了mybatis-plus的自带方法 + List methodList = super.getMethodList(mapperClass, tableInfo); + // 注入InsertBatchSomeColumn + // 在!t.isLogicDelete()表示不要逻辑删除字段，!"update_time".equals(t.getColumn())表示不要字段名为 update_time 的字段,不对进行操作 + // methodList.add(new InsertBatchSomeColumn(t -> !t.isLogicDelete() && !"update_time".equals(t.getColumn()))); + // 要逻辑删除 t.isLogicDelete() 默认不要 + methodList.add(new InsertBatchSomeColumn(t -> !t.isLogicDelete())); + return methodList; + } +}