caojiahao/dev_caojiahao
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

敏感词更新策略问题调整，敏感词迁移至公共工具类

Browse Source
This commit is contained in:
Chopper 2021-11-23 16:56:03 +08:00
parent f8408030a9
commit 16f314bd80
16 changed files with 160 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
buyer-api/src/main/java/cn/lili/controller/passport/connect/ConnectBuyerWebController.java
View File

@ -11,7 +11,7 @@ import cn.lili.modules.connect.entity.dto.ConnectAuthUser;
import cn.lili.modules.connect.request.AuthRequest;
import cn.lili.modules.connect.service.ConnectService;
import cn.lili.modules.connect.util.ConnectUtil;
import cn.lili.modules.connect.util.UuidUtils;
import cn.lili.common.utils.UuidUtils;
import cn.lili.modules.member.service.MemberService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;

26
framework/pom.xml
View File

@ -26,6 +26,12 @@
</exclusion>
</exclusions>
</dependency>
<!--定时任务-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-quartz</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
@ -74,11 +80,11 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-mongodb</artifactId>
</dependency>
<!-- &lt;!&ndash; Websocket &ndash;&gt;-->
<!-- <dependency>-->
<!-- <groupId>org.springframework.boot</groupId>-->
<!-- <artifactId>spring-boot-starter-websocket</artifactId>-->
<!-- </dependency>-->
<!-- &lt;!&ndash; Websocket &ndash;&gt;-->
<!-- <dependency>-->
<!-- <groupId>org.springframework.boot</groupId>-->
<!-- <artifactId>spring-boot-starter-websocket</artifactId>-->
<!-- </dependency>-->
<!-- MybatisPlus -->
<dependency>
<groupId>com.baomidou</groupId>
@ -265,11 +271,11 @@
<artifactId>logstash-logback-encoder</artifactId>
<version>${logstash-logback-encoder}</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>javax.interceptor</groupId>-->
<!-- <artifactId>javax.interceptor-api</artifactId>-->
<!-- <version>${interceptor-api}</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>javax.interceptor</groupId>-->
<!-- <artifactId>javax.interceptor-api</artifactId>-->
<!-- <version>${interceptor-api}</version>-->
<!-- </dependency>-->
<dependency>
<groupId>de.codecentric</groupId>
<artifactId>spring-boot-admin-starter-client</artifactId>

6
framework/src/main/java/cn/lili/cache/CachePrefix.java vendored
View File

@ -480,7 +480,11 @@ public enum CachePrefix {
/**
* 订单暂时缓存
*/
ORDER;
ORDER,
/**
* 敏感词
*/
SENSITIVE;
public static String removePrefix(String str) {

20
framework/src/main/java/cn/lili/common/sensitive/SensitiveWordsFilter.java
View File

@ -1,4 +1,4 @@
package cn.lili.modules.system.utils;
package cn.lili.common.sensitive;
import lombok.extern.slf4j.Slf4j;
@ -34,7 +34,12 @@ public class SensitiveWordsFilter implements Serializable {
* 类似HashMap的桶，比较稀疏。
* 使用2个字符的hash定位。
*/
protected static SensitiveWordsNode[] nodes;
protected static SensitiveWordsNode[] nodes = new SensitiveWordsNode[0];
/**
* 更新中的nodes，用于防止动态更新时，原有nodes被清空，导致无法正常写入过滤词
*/
protected static SensitiveWordsNode[] nodesUpdate;
/**
@ -150,13 +155,14 @@ public class SensitiveWordsFilter implements Serializable {
* 初始化敏感词
*/
public static void init(List<String> words) {
nodes = new SensitiveWordsNode[DEFAULT_INITIAL_CAPACITY];
log.info("开始初始化敏感词");
nodesUpdate = new SensitiveWordsNode[DEFAULT_INITIAL_CAPACITY];
for (String word : words) {
put(word);
}
nodes = nodesUpdate;
}
/**
* 增加一个敏感词，如果词的长度（trim后）小于2，则丢弃<br/>
* 此方法（构建）并不是主要的性能优化点。
@ -180,17 +186,17 @@ public class SensitiveWordsFilter implements Serializable {
//计算头两个字符的mix表示（mix相同，两个字符相同）
int mix = sp.nextTwoCharMix(0);
//转为在hash桶中的位置
int index = hash & (nodes.length - 1);
int index = hash & (nodesUpdate.length - 1);
//从桶里拿第一个节点
SensitiveWordsNode node = nodes[index];
SensitiveWordsNode node = nodesUpdate[index];
if (node == null) {
//如果没有节点，则放进去一个
node = new SensitiveWordsNode(mix);
//并添加词
node.words.add(sp);
//放入桶里
nodes[index] = node;
nodesUpdate[index] = node;
} else {
//如果已经有节点（1个或多个），找到正确的节点
for (; node != null; node = node.next) {

2
framework/src/main/java/cn/lili/common/sensitive/SensitiveWordsNode.java
View File

@ -1,4 +1,4 @@
package cn.lili.modules.system.utils;
package cn.lili.common.sensitive;
import java.io.Serializable;
import java.util.TreeSet;

48
framework/src/main/java/cn/lili/common/sensitive/StringPointer.java
View File

@ -1,13 +1,14 @@
package cn.lili.modules.system.utils;
package cn.lili.common.sensitive;
import java.io.Serializable;
/**
* 字符指针
*
* @author Bulbasaur
* @since 2020-02-25 14:10:16
*/
public class StringPointer implements Serializable, CharSequence, Comparable<StringPointer>{
public class StringPointer implements Serializable, CharSequence, Comparable<StringPointer> {
private static final long serialVersionUID = 1L;
@ -19,13 +20,13 @@ public class StringPointer implements Serializable, CharSequence, Comparable<Str
private int hash = 0;
public StringPointer(String str){
public StringPointer(String str) {
value = str.toCharArray();
offset = 0;
length = value.length;
}
public StringPointer(char[] value, int offset, int length){
public StringPointer(char[] value, int offset, int length) {
this.value = value;
this.offset = offset;
this.length = length;
@ -34,10 +35,11 @@ public class StringPointer implements Serializable, CharSequence, Comparable<Str
/**
* 计算该位置后（包含）2个字符的hash值
*
* @param i 从 0 到 length - 2
* @return 从 0 到 length - 2
*/
public int nextTwoCharHash(int i){
public int nextTwoCharHash(int i) {
return 31 * value[offset + i] + value[offset + i + 1];
}
@ -48,25 +50,25 @@ public class StringPointer implements Serializable, CharSequence, Comparable<Str
* @param i 从 0 到 length - 2
* @return int值
*/
public int nextTwoCharMix(int i){
public int nextTwoCharMix(int i) {
return (value[offset + i] << 16) | value[offset + i + 1];
}
/**
* 该位置后（包含）的字符串，是否以某个词（word）开头
*
* @param i 从 0 到 length - 2
* @param i 从 0 到 length - 2
* @param word 词
* @return 是否？
*/
public boolean nextStartsWith(int i, StringPointer word){
public boolean nextStartsWith(int i, StringPointer word) {
//是否长度超出
if(word.length > length - i){
if (word.length > length - i) {
return false;
}
//从尾开始判断
for(int c = word.length - 1; c >= 0; c --){
if(value[offset + i + c] != word.value[word.offset + c]){
for (int c = word.length - 1; c >= 0; c--) {
if (value[offset + i + c] != word.value[word.offset + c]) {
return false;
}
}
@ -76,31 +78,31 @@ public class StringPointer implements Serializable, CharSequence, Comparable<Str
/**
* 填充（替换）
*
* @param begin 从此位置开始（含）
* @param end 到此位置结束（不含）
* @param begin 从此位置开始（含）
* @param end 到此位置结束（不含）
* @param fillWith 以此字符填充（替换）
*/
public void fill(int begin, int end, char fillWith){
for(int i = begin; i < end; i ++){
public void fill(int begin, int end, char fillWith) {
for (int i = begin; i < end; i++) {
value[offset + i] = fillWith;
}
}
@Override
public int length(){
public int length() {
return length;
}
@Override
public char charAt(int i){
public char charAt(int i) {
return value[offset + i];
}
public StringPointer substring(int begin){
public StringPointer substring(int begin) {
return new StringPointer(value, offset + begin, length - begin);
}
public StringPointer substring(int begin, int end){
public StringPointer substring(int begin, int end) {
return new StringPointer(value, offset + begin, end - begin);
}
@ -110,7 +112,7 @@ public class StringPointer implements Serializable, CharSequence, Comparable<Str
}
@Override
public String toString(){
public String toString() {
return new String(value, offset, length);
}
@ -132,12 +134,12 @@ public class StringPointer implements Serializable, CharSequence, Comparable<Str
return true;
}
if (anObject instanceof StringPointer) {
StringPointer that = (StringPointer)anObject;
StringPointer that = (StringPointer) anObject;
if (length == that.length) {
char[] v1 = this.value;
char[] v2 = that.value;
for(int i = 0; i < this.length; i ++){
if(v1[this.offset + i] != v2[that.offset + i]){
for (int i = 0; i < this.length; i++) {
if (v1[this.offset + i] != v2[that.offset + i]) {
return false;
}
}

7
framework/src/main/java/cn/lili/common/sensitive/init/SensitiveInit.java
View File

@ -1,8 +1,9 @@
package cn.lili.cache.impl;
package cn.lili.common.sensitive.init;
import cn.lili.cache.Cache;
import cn.lili.cache.CachePrefix;
import cn.lili.common.sensitive.SensitiveWordsFilter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
@ -18,6 +19,7 @@ import java.util.List;
* 2021-11-23 12:08
*/
@Component
@Slf4j
public class SensitiveInit implements ApplicationRunner {
@Autowired
@ -26,11 +28,12 @@ public class SensitiveInit implements ApplicationRunner {
/**
* 程序启动时，获取最新的需要过滤的敏感词
*
* @param args
* @param args 启动参数
*/
@Override
public void run(ApplicationArguments args) {
List<String> sensitives = cache.get(CachePrefix.SENSITIVE.getPrefix());
log.info("系统初始化敏感词");
if (sensitives == null || sensitives.isEmpty()) {
return;
}

30
framework/src/main/java/cn/lili/common/sensitive/quartz/QuartzConfig.java
View File

@ -1,11 +1,31 @@
package cn.lili.common.sensitive.quartz;
import org.quartz.*;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
*
* QuartzConfig
* 定时执行配置
*
* @author Chopper
* @version v1.0
* 2021-11-23 16:30
*
*/
*/
@Configuration
public class QuartzConfig {
}
@Bean
public JobDetail sensitiveQuartzDetail() {
return JobBuilder.newJob(SensitiveQuartz.class).withIdentity("sensitiveQuartz").storeDurably().build();
}
@Bean
public Trigger sensitiveQuartzTrigger() {
SimpleScheduleBuilder scheduleBuilder = SimpleScheduleBuilder.simpleSchedule()
.withIntervalInSeconds(3600)
.repeatForever();
return TriggerBuilder.newTrigger().forJob(sensitiveQuartzDetail())
.withIdentity("sensitiveQuartz")
.withSchedule(scheduleBuilder)
.build();
}
}

42
framework/src/main/java/cn/lili/common/sensitive/quartz/SensitiveQuartz.java
View File

@ -1,11 +1,41 @@
package cn.lili.common.sensitive.quartz;
import cn.lili.cache.Cache;
import cn.lili.cache.CachePrefix;
import cn.lili.common.sensitive.SensitiveWordsFilter;
import lombok.extern.slf4j.Slf4j;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.quartz.QuartzJobBean;
import java.util.List;
/**
*
* SensitiveQuartz
* 间隔更新敏感词
*
* @author Chopper
* @version v1.0
* 2021-11-23 16:31
*
*/
public class SensitiveQuartz {
}
*/
@Slf4j
public class SensitiveQuartz extends QuartzJobBean {
@Autowired
private Cache<List<String>> cache;
/**
* 定时更新敏感词信息
*
* @param jobExecutionContext
*/
@Override
protected void executeInternal(JobExecutionContext jobExecutionContext) {
log.info("敏感词定时更新");
List<String> sensitives = cache.get(CachePrefix.SENSITIVE.getPrefix());
if (sensitives == null || sensitives.isEmpty()) {
return;
}
SensitiveWordsFilter.init(sensitives);
}
}

2
framework/src/main/java/cn/lili/common/utils/UuidUtils.java
View File

@ -1,4 +1,4 @@
package cn.lili.modules.connect.util;
package cn.lili.common.utils;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.ThreadLocalRandom;

2
framework/src/main/java/cn/lili/modules/connect/request/BaseAuthRequest.java
View File

@ -13,7 +13,7 @@ import cn.lili.modules.connect.entity.enums.AuthResponseStatus;
import cn.lili.modules.connect.exception.AuthException;
import cn.lili.modules.connect.util.AuthChecker;
import cn.lili.common.utils.HttpUtils;
import cn.lili.modules.connect.util.UuidUtils;
import cn.lili.common.utils.UuidUtils;
import com.xkcoding.http.util.UrlUtil;
import lombok.extern.slf4j.Slf4j;

5
framework/src/main/java/cn/lili/modules/member/serviceimpl/MemberEvaluationServiceImpl.java
View File

@ -31,8 +31,7 @@ import cn.lili.modules.order.order.entity.dos.OrderItem;
import cn.lili.modules.order.order.entity.enums.CommentStatusEnum;
import cn.lili.modules.order.order.service.OrderItemService;
import cn.lili.modules.order.order.service.OrderService;
import cn.lili.modules.system.utils.CharacterConstant;
import cn.lili.modules.system.utils.SensitiveWordsFilter;
import cn.lili.common.sensitive.SensitiveWordsFilter;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
@ -121,7 +120,7 @@ public class MemberEvaluationServiceImpl extends ServiceImpl<MemberEvaluationMap
//新增用户评价
MemberEvaluation memberEvaluation = new MemberEvaluation(memberEvaluationDTO, goodsSku, member, order);
//过滤商品咨询敏感词
memberEvaluation.setContent(SensitiveWordsFilter.filter(memberEvaluation.getContent(), CharacterConstant.WILDCARD_STAR));
memberEvaluation.setContent(SensitiveWordsFilter.filter(memberEvaluation.getContent()));
//添加评价
this.save(memberEvaluation);

7
framework/src/main/java/cn/lili/modules/member/serviceimpl/MemberServiceImpl.java
View File

@ -22,7 +22,7 @@ import cn.lili.modules.connect.config.ConnectAuthEnum;
import cn.lili.modules.connect.entity.Connect;
import cn.lili.modules.connect.entity.dto.ConnectAuthUser;
import cn.lili.modules.connect.service.ConnectService;
import cn.lili.modules.connect.util.UuidUtils;
import cn.lili.common.utils.UuidUtils;
import cn.lili.modules.member.aop.annotation.PointLogPoint;
import cn.lili.modules.member.entity.dos.Member;
import cn.lili.modules.member.entity.dto.ManagerMemberEditDTO;
@ -40,8 +40,7 @@ import cn.lili.modules.member.token.StoreTokenGenerate;
import cn.lili.modules.store.entity.dos.Store;
import cn.lili.modules.store.entity.enums.StoreStatusEnum;
import cn.lili.modules.store.service.StoreService;
import cn.lili.modules.system.utils.CharacterConstant;
import cn.lili.modules.system.utils.SensitiveWordsFilter;
import cn.lili.common.sensitive.SensitiveWordsFilter;
import cn.lili.mybatis.util.PageUtil;
import cn.lili.rocketmq.RocketmqSendCallbackBuilder;
import cn.lili.rocketmq.tags.MemberTagsEnum;
@ -351,7 +350,7 @@ public class MemberServiceImpl extends ServiceImpl<MemberMapper, Member> impleme
}
//过滤会员昵称敏感词
if (com.baomidou.mybatisplus.core.toolkit.StringUtils.isNotBlank(managerMemberEditDTO.getNickName())) {
managerMemberEditDTO.setNickName(SensitiveWordsFilter.filter(managerMemberEditDTO.getNickName(), CharacterConstant.WILDCARD_STAR));
managerMemberEditDTO.setNickName(SensitiveWordsFilter.filter(managerMemberEditDTO.getNickName()));
}
//如果密码不为空则加密密码
if (com.baomidou.mybatisplus.core.toolkit.StringUtils.isNotBlank(managerMemberEditDTO.getPassword())) {

4
framework/src/main/java/cn/lili/modules/system/service/SensitiveWordsService.java
View File

@ -11,5 +11,9 @@ import com.baomidou.mybatisplus.extension.service.IService;
*/
public interface SensitiveWordsService extends IService<SensitiveWords> {
/**
* 重新写入缓存
*/
void resetCache();
}

17
framework/src/main/java/cn/lili/modules/system/serviceimpl/SensitiveWordsServiceImpl.java
View File

@ -1,5 +1,7 @@
package cn.lili.modules.system.serviceimpl;
import cn.lili.cache.Cache;
import cn.lili.cache.CachePrefix;
import cn.lili.modules.system.entity.dos.SensitiveWords;
import cn.lili.modules.system.mapper.SensitiveWordsMapper;
import cn.lili.modules.system.service.SensitiveWordsService;
@ -9,6 +11,9 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
import java.util.stream.Collectors;
/**
* 敏感词业务层实现
*
@ -17,5 +22,17 @@ import org.springframework.transaction.annotation.Transactional;
*/
@Service
public class SensitiveWordsServiceImpl extends ServiceImpl<SensitiveWordsMapper, SensitiveWords> implements SensitiveWordsService {
@Autowired
private Cache<List<String>> cache;
@Override
public void resetCache() {
List<SensitiveWords> sensitiveWordsList = this.list();
if (sensitiveWordsList == null || sensitiveWordsList.isEmpty()) {
return;
}
List<String> sensitiveWords = sensitiveWordsList.stream().map(SensitiveWords::getSensitiveWord).collect(Collectors.toList());
cache.put(CachePrefix.SENSITIVE.getPrefix(), sensitiveWords);
}
}

14
manager-api/src/main/java/cn/lili/controller/other/SensitiveWordsManagerController.java
View File

@ -6,7 +6,7 @@ import cn.lili.common.vo.PageVO;
import cn.lili.common.vo.ResultMessage;
import cn.lili.modules.system.entity.dos.SensitiveWords;
import cn.lili.modules.system.service.SensitiveWordsService;
import cn.lili.modules.system.utils.SensitiveWordsFilter;
import cn.lili.common.sensitive.SensitiveWordsFilter;
import com.baomidou.mybatisplus.core.metadata.IPage;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
@ -48,7 +48,7 @@ public class SensitiveWordsManagerController {
@PostMapping
public ResultMessage<SensitiveWords> add(@Valid SensitiveWords sensitiveWords) {
sensitiveWordsService.save(sensitiveWords);
SensitiveWordsFilter.put(sensitiveWords.getSensitiveWord());
sensitiveWordsService.resetCache();
return ResultUtil.data(sensitiveWords);
}
@ -58,7 +58,7 @@ public class SensitiveWordsManagerController {
public ResultMessage<SensitiveWords> edit(@PathVariable String id, SensitiveWords sensitiveWords) {
sensitiveWords.setId(id);
sensitiveWordsService.updateById(sensitiveWords);
SensitiveWordsFilter.put(sensitiveWords.getSensitiveWord());
sensitiveWordsService.resetCache();
return ResultUtil.data(sensitiveWords);
}
@ -66,12 +66,8 @@ public class SensitiveWordsManagerController {
@ApiImplicitParam(name = "ids", value = "敏感词ID", required = true, dataType = "String", allowMultiple = true, paramType = "path")
@DeleteMapping(value = "/delByIds/{ids}")
public ResultMessage<Object> delAllByIds(@PathVariable List<String> ids) {
for (String id : ids) {
String name = sensitiveWordsService.getById(id).getSensitiveWord();
SensitiveWordsFilter.remove(name);
sensitiveWordsService.removeById(id);
}
sensitiveWordsService.removeByIds(ids);
sensitiveWordsService.resetCache();
return ResultUtil.success();
}
}