!111 优化es商品索引的mapping的促销存储结构,增加自定义xss过滤策略

Merge pull request !111 from OceansDeep/feature/pg

consumer/src/main/java/cn/lili/listener/GoodsMessageListener.java

@@ -390,7 +390,7 @@ public class GoodsMessageListener implements RocketMQListener<MessageExt> {
 
         if (goodsIndex.getPromotionMap() == null || goodsIndex.getPromotionMap().isEmpty()) {
             Map<String, Object> goodsCurrentPromotionMap = promotionService.getGoodsPromotionMap(goodsIndex);
-            goodsIndex.setPromotionMap(goodsCurrentPromotionMap);
+            goodsIndex.setPromotionMapJson(JSONUtil.toJsonStr(goodsCurrentPromotionMap));
         }
     }
 

framework/src/main/java/cn/lili/cache/config/redis/RedisConfig.java

@@ -78,6 +78,7 @@ public class RedisConfig extends CachingConfigurerSupport {
         可参考 https://blog.csdn.net/u012240455/article/details/80538540
          */
         ParserConfig.getGlobalInstance().addAccept("cn.lili.");
+        ParserConfig.getGlobalInstance().addAccept("cn.hutool.json.");
 
         return cacheManager;
     }

framework/src/main/java/cn/lili/common/enums/PromotionTypeEnum.java

@@ -23,7 +23,7 @@ public enum PromotionTypeEnum {
     /**
      * 有促销库存的活动类型
      */
-    static PromotionTypeEnum[] haveStockPromotion = new PromotionTypeEnum[]{PINTUAN, SECKILL, KANJIA, POINTS_GOODS};
+    static final PromotionTypeEnum[] haveStockPromotion = new PromotionTypeEnum[]{PINTUAN, SECKILL, KANJIA, POINTS_GOODS};
 
     private final String description;
 

framework/src/main/java/cn/lili/common/security/filter/XssHttpServletRequestWrapper.java

@@ -5,6 +5,8 @@ import cn.hutool.core.text.CharSequenceUtil;
 import cn.hutool.http.HtmlUtil;
 import cn.hutool.json.JSONUtil;
 import lombok.extern.slf4j.Slf4j;
+import org.owasp.html.HtmlPolicyBuilder;
+import org.owasp.html.PolicyFactory;
 import org.owasp.html.Sanitizers;
 
 import javax.servlet.ReadListener;
@@ -57,6 +59,24 @@ public class XssHttpServletRequestWrapper extends HttpServletRequestWrapper {
             "wechatpay",
     };
 
+    //允许的标签
+    private static final String[] allowedTags = {"h1", "h2", "h3", "h4", "h5", "h6",
+            "span", "strong",
+            "img", "video", "source", "iframe", "code",
+            "blockquote", "p", "div",
+            "ul", "ol", "li",
+            "table", "thead", "caption", "tbody", "tr", "th", "td", "br",
+            "a"
+    };
+
+    //需要转化的标签
+    private static final String[] needTransformTags = {"article", "aside", "command", "datalist", "details", "figcaption", "figure",
+            "footer", "header", "hgroup", "section", "summary"};
+
+    //带有超链接的标签
+    private static final String[] linkTags = {"img", "video", "source", "a", "iframe"};
+
+
     public XssHttpServletRequestWrapper(HttpServletRequest request) {
         super(request);
     }
@@ -257,6 +277,17 @@ public class XssHttpServletRequestWrapper extends HttpServletRequestWrapper {
 
     private String cleanXSS(String value) {
         if (value != null) {
+            // 自定义策略
+            PolicyFactory policy = new HtmlPolicyBuilder()
+                    .allowStandardUrlProtocols()
+                    //所有允许的标签
+                    .allowElements(allowedTags)
+                    //内容标签转化为div
+                    .allowElements((elementName, attributes) -> "div", needTransformTags)
+                    .allowAttributes("src", "href", "target", "width", "height").onElements(linkTags)
+                    //校验链接中的是否为http
+//                    .allowUrlProtocols("https")
+                    .toFactory();
             // basic prepackaged policies for links, tables, integers, images, styles, blocks
             value = Sanitizers.FORMATTING
                     .and(Sanitizers.STYLES)
@@ -264,6 +295,7 @@ public class XssHttpServletRequestWrapper extends HttpServletRequestWrapper {
                     .and(Sanitizers.LINKS)
                     .and(Sanitizers.BLOCKS)
                     .and(Sanitizers.TABLES)
+                    .and(policy)
                     .sanitize(value);
         }
         return HtmlUtil.unescape(value);

framework/src/main/java/cn/lili/elasticsearch/BaseElasticsearchService.java

@@ -331,6 +331,9 @@ public abstract class BaseElasticsearchService {
                         "          }\n" +
                         "        }\n" +
                         "      },\n" +
+                        "      \"promotionMapJson\": {\n" +
+                        "        \"type\": \"text\"\n" +
+                        "      },\n" +
                         "      \"thumbnail\": {\n" +
                         "        \"type\": \"text\",\n" +
                         "        \"fields\": {\n" +

framework/src/main/java/cn/lili/modules/goods/serviceimpl/GoodsSkuServiceImpl.java

@@ -4,7 +4,6 @@ import cn.hutool.core.convert.Convert;
 import cn.hutool.core.map.MapUtil;
 import cn.hutool.core.text.CharSequenceUtil;
 import cn.hutool.core.util.NumberUtil;
-import cn.hutool.core.util.StrUtil;
 import cn.hutool.json.JSONObject;
 import cn.hutool.json.JSONUtil;
 import cn.lili.cache.Cache;
@@ -575,16 +574,6 @@ public class GoodsSkuServiceImpl extends ServiceImpl<GoodsSkuMapper, GoodsSku> i
         String destination = rocketmqCustomProperties.getGoodsTopic() + ":" + GoodsTagsEnum.GENERATOR_GOODS_INDEX.name();
         //发送mq消息
         rocketMQTemplate.asyncSend(destination, goods.getId(), RocketmqSendCallbackBuilder.commonCallback());
-//        ThreadUtil.execAsync(() -> {
-//            try {
-//                // 延时执行，防止商品未保存完成就去生成商品索引导致生成索引时找不到商品问题
-//                Thread.sleep(2000);
-//
-//            } catch (InterruptedException e) {
-//                log.error("发送商品索引信息失败！", e);
-//                Thread.currentThread().interrupt();
-//            }
-//        });
     }
 
     /**
@@ -719,7 +708,7 @@ public class GoodsSkuServiceImpl extends ServiceImpl<GoodsSkuMapper, GoodsSku> i
                     }
                     //设置规格商品缩略图
                     //如果规格没有图片，则用商品图片复盖。有则增加规格图片，放在商品图片集合之前
-                    if (spec.getValue() != null && StrUtil.isNotEmpty(spec.getValue().toString())) {
+                    if (CharSequenceUtil.isNotEmpty(spec.getValue().toString())) {
                         thumbnail = goodsGalleryService.getGoodsGallery(images.get(0).get("url")).getThumbnail();
                         small = goodsGalleryService.getGoodsGallery(images.get(0).get("url")).getSmall();
                     }

framework/src/main/java/cn/lili/modules/search/entity/dos/EsGoodsIndex.java

@@ -1,5 +1,6 @@
 package cn.lili.modules.search.entity.dos;
 
+import cn.hutool.json.JSONUtil;
 import cn.lili.common.enums.PromotionTypeEnum;
 import cn.lili.elasticsearch.EsSuffix;
 import cn.lili.modules.goods.entity.dos.GoodsSku;
@@ -260,9 +261,6 @@ public class EsGoodsIndex implements Serializable {
     @ApiModelProperty(value = "商品类型", required = true)
     private String goodsType;
 
-    /**
-     * @see cn.lili.modules.goods.entity.enums.GoodsTypeEnum
-     */
     @ApiModelProperty(value = "商品sku基础分数", required = true)
     private Integer skuSource;
 
@@ -280,8 +278,8 @@ public class EsGoodsIndex implements Serializable {
      * value 为 促销活动实体信息
      */
     @Field(type = FieldType.Nested)
-    @ApiModelProperty("商品促销活动集合，key 为 促销活动类型，value 为 促销活动实体信息 ")
+    @ApiModelProperty("商品促销活动集合JSON，key 为 促销活动类型，value 为 促销活动实体信息 ")
-    private Map<String, Object> promotionMap;
+    private String promotionMapJson;
 
 
     public EsGoodsIndex(GoodsSku sku) {
@@ -381,6 +379,6 @@ public class EsGoodsIndex implements Serializable {
     }
 
     public Map<String, Object> getPromotionMap() {
-        return PromotionTools.filterInvalidPromotionsMap(this.promotionMap);
+        return PromotionTools.filterInvalidPromotionsMap(JSONUtil.parseObj(this.promotionMapJson));
     }
 }

framework/src/main/java/cn/lili/modules/search/serviceimpl/EsGoodsIndexServiceImpl.java

@@ -46,6 +46,7 @@ import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
 import org.elasticsearch.index.reindex.UpdateByQueryRequest;
 import org.elasticsearch.script.Script;
+import org.elasticsearch.script.ScriptType;
 import org.mybatis.spring.MyBatisSystemException;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
@@ -451,7 +452,7 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
                     //如果存在同类型促销活动删除
                     List<String> collect = promotionMap.keySet().stream().filter(i -> i.contains(promotionType.name())).collect(Collectors.toList());
                     collect.forEach(promotionMap::remove);
-                    goodsIndex.setPromotionMap(promotionMap);
+                    goodsIndex.setPromotionMapJson(JSONUtil.toJsonStr(promotionMap));
                     updateIndex(goodsIndex);
                 }
             } else {
@@ -503,7 +504,7 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
             //如果存在同促销ID的活动删除
             List<String> collect = promotionMap.keySet().stream().filter(i -> i.split("-")[1].equals(promotionId)).collect(Collectors.toList());
             collect.forEach(promotionMap::remove);
-            goodsIndex.setPromotionMap(promotionMap);
+            goodsIndex.setPromotionMapJson(JSONUtil.toJsonStr(promotionMap));
             return this.getGoodsIndexPromotionUpdateRequest(goodsIndex.getId(), promotionMap);
         }
         return null;
@@ -598,7 +599,7 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
         //获取活动信息
         Map<String, Object> goodsCurrentPromotionMap = promotionService.getGoodsPromotionMap(index);
         //写入促销信息
-        index.setPromotionMap(goodsCurrentPromotionMap);
+        index.setPromotionMapJson(JSONUtil.toJsonStr(goodsCurrentPromotionMap));
         return index;
     }
 
@@ -634,16 +635,14 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
      * @param promotionMap 促销信息
      */
     private UpdateRequest getGoodsIndexPromotionUpdateRequest(String id, Map<String, Object> promotionMap) {
-        JSONObject jsonObject = JSONUtil.parseObj(promotionMap);
-        jsonObject.setDateFormat("yyyy-MM-dd HH:mm:ss");
-        String s = jsonObject.toString();
-        String promotionsStr = s.replace("{", "[").replace("}", "]");
-
         UpdateRequest updateRequest = new UpdateRequest();
         updateRequest.index(getIndexName());
         updateRequest.id(id);
         updateRequest.retryOnConflict(5);
-        updateRequest.script(new Script("ctx._source." + "promotionMap" + "=" + promotionsStr + ";"));
+        Map<String, Object> params = new HashMap<>();
+        params.put("promotionMap", JSONUtil.toJsonStr(promotionMap));
+        Script script = new Script(ScriptType.INLINE, "painless", "ctx._source.promotionMapJson=params.promotionMap;", params);
+        updateRequest.script(script);
         return updateRequest;
     }
 
@@ -758,7 +757,7 @@ public class EsGoodsIndexServiceImpl extends BaseElasticsearchService implements
         }
         //促销索引
         Map<String, Object> goodsCurrentPromotionMap = promotionService.getGoodsPromotionMap(index);
-        index.setPromotionMap(goodsCurrentPromotionMap);
+        index.setPromotionMapJson(JSONUtil.toJsonStr(goodsCurrentPromotionMap));
         return index;
     }