Java DFA sensitive words filter Java实现的敏感词过滤(支持停顿词/重复词/全半角/字符串替换，改进https://www.jb51.net/article/128990.htm)

BCConvert.java

package com.haofly.net.sensitiveword;


public class BCConvert {

	/**
	 * ASCII表中可见字符从!开始，偏移位值为33(Decimal)
	 */
	static final char DBC_CHAR_START = 33; // 半角!

	/**
	 * ASCII表中可见字符到~结束，偏移位值为126(Decimal)
	 */
	static final char DBC_CHAR_END = 126; // 半角~

	/**
	 * 全角对应于ASCII表的可见字符从！开始，偏移值为65281
	 */
	static final char SBC_CHAR_START = 65281; // 全角！

	/**
	 * 全角对应于ASCII表的可见字符到～结束，偏移值为65374
	 */
	static final char SBC_CHAR_END = 65374; // 全角～

	/**
	 * ASCII表中除空格外的可见字符与对应的全角字符的相对偏移
	 */
	static final int CONVERT_STEP = 65248; // 全角半角转换间隔

	/**
	 * 全角空格的值，它没有遵从与ASCII的相对偏移，必须单独处理
	 */
	static final char SBC_SPACE = 12288; // 全角空格 12288

	/**
	 * 半角空格的值，在ASCII中为32(Decimal)
	 */
	static final char DBC_SPACE = ' '; // 半角空格

	/**
	 * <PRE>
	 * 半角字符->全角字符转换  
	 * 只处理空格，!到˜之间的字符，忽略其他
	 * </PRE>
	 */
	public static String bj2qj(String src) {
		if (src == null) {
			return src;
		}
		StringBuilder buf = new StringBuilder(src.length());
		char[] ca = src.toCharArray();
		for (int i = 0; i < ca.length; i++) {
			if (ca[i] == DBC_SPACE) { // 如果是半角空格，直接用全角空格替代
				buf.append(SBC_SPACE);
			} else if ((ca[i] >= DBC_CHAR_START) && (ca[i] <= DBC_CHAR_END)) { // 字符是!到~之间的可见字符
				buf.append((char) (ca[i] + CONVERT_STEP));
			} else { // 不对空格以及ascii表中其他可见字符之外的字符做任何处理
				buf.append(ca[i]);
			}
		}
		return buf.toString();
	}

	/**
	 * 半角转换全角
	 * 
	 * @param src
	 * @return
	 */
	public static int bj2qj(char src) {
		int r = src;
		if (src == DBC_SPACE) { // 如果是半角空格，直接用全角空格替代
			src = SBC_SPACE;
		} else if ((src >= DBC_CHAR_START) && (src <= DBC_CHAR_END)) { // 字符是!到~之间的可见字符
			r = src + CONVERT_STEP;
		}
		return r;
	}

	/**
	 * <PRE>
	 * 全角字符->半角字符转换  
	 * 只处理全角的空格，全角！到全角～之间的字符，忽略其他
	 * </PRE>
	 */
	public static String qj2bj(String src) {
		if (src == null) {
			return src;
		}
		StringBuilder buf = new StringBuilder(src.length());
		char[] ca = src.toCharArray();
		for (int i = 0; i < src.length(); i++) {
			if (ca[i] >= SBC_CHAR_START && ca[i] <= SBC_CHAR_END) { // 如果位于全角！到全角～区间内
				buf.append((char) (ca[i] - CONVERT_STEP));
			} else if (ca[i] == SBC_SPACE) { // 如果是全角空格
				buf.append(DBC_SPACE);
			} else { // 不处理全角空格，全角！到全角～区间外的字符
				buf.append(ca[i]);
			}
		}
		return buf.toString();
	}

	/**
	 * 全角转换半角
	 * 
	 * @param src
	 * @return
	 */
	public static int qj2bj(char src) {
		int r = src;
		if (src >= SBC_CHAR_START && src <= SBC_CHAR_END) { // 如果位于全角！到全角～区间内
			r = src - CONVERT_STEP;
		} else if (src == SBC_SPACE) { // 如果是全角空格
			r = DBC_SPACE;
		}
		return r;
	}

}

FileSet.java

package com.haofly.net.sensitiveword;

public class FilterSet{

	private final long[] elements;

	
	public FilterSet() {
		 elements = new long[1 + (65535 >>> 6)];
	}
	
	public void add(final int no) {
		elements[no >>> 6] |= (1L << (no & 63));
	}
	
	public void add(final int... no) {
		for(int currNo : no) {
			elements[currNo >>> 6] |= (1L << (currNo & 63));
		}
	}
	
	public void remove(final int no) {
		elements[no >>> 6] &= ~(1L << (no & 63));
	}
	
	/**
	 * 
	 * @param no
	 * @return true:添加成功	false:原已包含
	 */
	public boolean addAndNotify(final int no) {
		int eWordNum = no >>> 6;
		long oldElements = elements[eWordNum];
		elements[eWordNum] |= (1L << (no & 63));
		boolean result = elements[eWordNum] != oldElements;
//		if (result)
//			size++;
		return result;
	}
	
	/**
	 * 
	 * @param no
	 * @return true:移除成功	false:原本就不包含
	 */
	public boolean removeAndNotify(final int no) {
		int eWordNum = no >>> 6;
		long oldElements = elements[eWordNum];
		elements[eWordNum] &= ~(1L << (no & 63));
		boolean result = elements[eWordNum] != oldElements;
		return result;
	}
	
	public boolean contains(final int no) {
        return (elements[no >>> 6] & (1L << (no & 63))) != 0;
    }

	public boolean containsAll(final int... no) {
		if(no.length==0) {
			return true;
		}
		for(int currNo : no) {
			if ((elements[currNo >>> 6] & (1L << (currNo & 63))) == 0) {
				return false;
			}
		}
		return true;
	}
	
	/**
	 * 不如直接循环调用contains
	 * @param no
	 * @return
	 */
	public boolean containsAll_ueslessWay(final int... no) {
		long[] elements = new long[this.elements.length];
		for(int currNo : no){
			elements[currNo >>> 6] |= (1L << (currNo & 63));
		}//这一步执行完跟循环调用contains差不多了
		
		for (int i = 0; i < elements.length; i++) {
			if ((elements[i] & ~this.elements[i]) != 0) {
				return false;
			}
		}
		return true;
	}
	
	/**
	 * 目前没有去维护size，每次都是去计算size
	 * @return
	 */
	public int size() {
		int size = 0;
		for (long element : elements) {
			size += Long.bitCount(element);
		}
		return size;
	}
	
	public static void main(String[] args) {
		FilterSet oi = new FilterSet();
		System.out.println(oi.elements.length);
	}

}

SensitiveContentDto.java

package com.haofly.net.sensitiveword;

import java.io.Serializable;
import java.util.Set;

public class SensitiveContentDto implements Serializable {
    private static final long serialVersionUID = 3487911794259456000L;

    private String oldContent;

    private String newContent;

    /**
     * 是否包含敏感词
     */
    private Boolean isContains;

    /**
     * 内容中出现的敏感词的列表
     */
    private Set<String> sensitiveWords;

    public String getOldContent() {
        return oldContent;
    }

    public void setOldContent(String oldContent) {
        this.oldContent = oldContent;
    }

    public String getNewContent() {
        return newContent;
    }

    public void setNewContent(String newContent) {
        this.newContent = newContent;
    }

    public Set<String> getSensitiveWords() {
        return sensitiveWords;
    }

    public void setSensitiveWords(Set<String> sensitiveWords) {
        this.sensitiveWords = sensitiveWords;
    }

    public Boolean getContains() {
        return isContains;
    }

    public void setContains(Boolean contains) {
        isContains = contains;
    }

    @Override
    public String toString() {
        return "SensitiveContentDto{" +
                "oldContent='" + oldContent + '\'' +
                ", newContent='" + newContent + '\'' +
                ", isContains=" + isContains +
                ", sensitiveWords=" + sensitiveWords +
                '}';
    }
}

SensitiveWordUtil.java

package com.haofly.net.sensitiveword;

import java.io.*;
import java.util.*;

public class SensitiveWordUtil {
    /**
     * 存储首字
     */
    private FilterSet set = new FilterSet();
    /**
     * 存储节点
     */
    private Map<Integer, WordNode> nodes = new HashMap<>(1024, 1);
    /**
     * 存储停顿词
     */
    private Set<Integer> stopWordSet = new HashSet<>();
    /**
     * 敏感词过滤替换
     */
    private char[] replaceChar;

    public SensitiveWordUtil() {
    }

    public SensitiveWordUtil(List<String> words) {
        addSensitiveWord(words);
    }

    public SensitiveWordUtil(List<String> words, List<String> stopWords, String replaceWord) {
        addSensitiveWord(words);
        addStopWord(stopWords);
        this.replaceChar = replaceWord.toCharArray();
    }

    /**
     * 从文件增加敏感词
     */
    private List<String> readWordFromFile(String path) {
        List<String> words;
        BufferedReader br = null;
        try {
            File file = new File(path);
            BufferedReader reader;
            reader = new BufferedReader(new FileReader(file));
            String tempString;
            int line = 1;
            words = new ArrayList<String>(1200);
            while ((tempString = reader.readLine()) != null) {
                if ("".equals(tempString.trim())) {
                    continue;
                }
                words.add(tempString.trim());
                line++;
            }
            reader.close();
        } catch (Exception e) {
            throw new RuntimeException(e);
        } finally {
            try {
                if (br != null) {
                    br.close();
                }
            } catch (IOException e) {
            }
        }
        return words;
    }

    /**
     * 添加停顿词
     */
    public void addStopWord(List<String> words) {
        if (words != null && words.size() > 0) {
            char[] chs;
            for (String curr : words) {
                chs = curr.toCharArray();
                for (char c : chs) {
                    stopWordSet.add(charConvert(c));
                }
            }
        }
    }

    /**
     * 添加敏感词/添加DFA节点
     */
    public void addSensitiveWord(List<String> words) {
        if (words != null && words.size() > 0) {
            char[] chs;
            int fchar;
            int lastIndex;
            WordNode fnode; // 首字母节点
            for (String curr : words) {
                chs = curr.toCharArray();
                fchar = charConvert(chs[0]);
                // 没有首字定义
                if (!set.contains(fchar)) {
                    // 首字标志位 可重复add,反正判断了，不重复了
                    set.add(fchar);
                    fnode = new WordNode(fchar, chs.length == 1);
                    nodes.put(fchar, fnode);
                } else {
                    fnode = nodes.get(fchar);
                    if (!fnode.isLast() && chs.length == 1) {
                        fnode.setLast(true);
                    }
                }
                lastIndex = chs.length - 1;
                for (int i = 1; i < chs.length; i++) {
                    fnode = fnode.addIfNoExist(charConvert(chs[i]), i == lastIndex);
                }
            }
        }
    }


    /**
     * 过滤判断 将敏感词转化为成屏蔽词
     */
    public SensitiveContentDto replaceSensitiveWord(String src) {
        char[] chs = src.toCharArray();
        int length = chs.length;
        int current;
        int k;
        WordNode node;

        Set<String> sensitiveWords = new HashSet<>();
        for (int i = 0; i < length; i++) {
            current = charConvert(chs[i]);
            if (!set.contains(current)) {
                continue;
            }
            node = nodes.get(current);
            // 其实不会发生，习惯性写上了
            if (node == null) {
                continue;
            }
            boolean couldMark = false;
            int markNum = -1;
            // 单字匹配（日）
            if (node.isLast()) {
                couldMark = true;
                markNum = 0;
                sensitiveWords.add(src);
            }
            // 继续匹配（日你/日你妹），以长的优先
            // 你-3 妹-4 夫-5
            k = i;
            for (; ++k < length; ) {
                int temp = charConvert(chs[k]);
                if (stopWordSet.contains(temp)) {
                    continue;
                }
                node = node.querySub(temp);
                if (node == null) {
                    break;
                }
                if (node.isLast()) {
                    couldMark = true;
                    markNum = k - i;
                    sensitiveWords.add(new String(Arrays.copyOfRange(chs, i, k + 1)));
                }
            }
            if (couldMark) {
                chs = this.modify(chs, replaceChar, i, markNum + 1);
                i = i - (markNum + 1) + replaceChar.length + 1;
                if (i < 0) {
                    break;
                }
                length = chs.length;
            }
        }

        SensitiveContentDto sensitiveContentDto = new SensitiveContentDto();
        sensitiveContentDto.setOldContent(src);
        sensitiveContentDto.setNewContent(Arrays.toString(chs));
        sensitiveContentDto.setSensitiveWords(sensitiveWords);
        sensitiveContentDto.setContains(sensitiveWords.size() > 0);
        return sensitiveContentDto;
    }

    /**
     * 是否包含敏感词
     */
    public boolean isContains(String src) {
        char[] chs = src.toCharArray();
        int length = chs.length;
        int currc;
        int k;
        WordNode node;
        for (int i = 0; i < length; i++) {
            currc = charConvert(chs[i]);
            if (!set.contains(currc)) {
                continue;
            }
            // 日 2
            node = nodes.get(currc);
            // 其实不会发生，习惯性写上了
            if (node == null) {
                continue;
            }
            boolean couldMark = false;
            // 单字匹配（日）
            if (node.isLast()) {
                couldMark = true;
            }
            // 继续匹配（日你/日你妹），以长的优先
            // 你-3 妹-4 夫-5
            k = i;
            for (; ++k < length; ) {
                int temp = charConvert(chs[k]);
                if (stopWordSet.contains(temp)) {
                    continue;
                }
                node = node.querySub(temp);
                if (node == null) {
                    break;
                }
                if (node.isLast()) {
                    couldMark = true;
                }
            }
            if (couldMark) {
                return true;
            }
        }

        return false;
    }

    /**
     * 大写转化为小写 全角转化为半角
     */
    private int charConvert(char src) {
        int r = BCConvert.qj2bj(src);
        return (r >= 'A' && r <= 'Z') ? r + 32 : r;
    }

    /**
     * @param chs     原始数组
     * @param replace 替换数组
     * @param index   替换位置
     * @param num     替换长度
     * @return 新的i的值
     */
    private char[] modify(char[] chs, char[] replace, int index, int num) {
        char[] temp = new char[chs.length - num + replace.length];
        for (int i = 0; i < temp.length; i++) {
            if (i < index) {
                temp[i] = chs[i];
            } else if (i == index) {
                for (int k = 0; k < replace.length; k++) {
                    temp[i + k] = replace[k];
                }
                i = i + replace.length - 1;
            } else {
                temp[i] = chs[i - replace.length + num];
            }
        }
        return temp;
    }
}

WordNode.java

package com.haofly.net.sensitiveword;

import java.util.LinkedList;
import java.util.List;

public class WordNode {

    private int value; // 节点名称

    private List<WordNode> subNodes; // 子节点

    private boolean isLast;// 默认false

    public WordNode(int value) {
        this.value = value;
    }

    public WordNode(int value, boolean isLast) {
        this.value = value;
        this.isLast = isLast;
    }

    /**
     *
     * @param subNode
     * @return 就是传入的subNode
     */
    private WordNode addSubNode(final WordNode subNode) {
        if (subNodes == null) {
            subNodes = new LinkedList<WordNode>();
        }
        subNodes.add(subNode);
        return subNode;
    }

    /**
     * 有就直接返回该子节点， 没有就创建添加并返回该子节点
     *
     * @param value
     * @return
     */
    public WordNode addIfNoExist(final int value, final boolean isLast) {
        if (subNodes == null) {
            return addSubNode(new WordNode(value, isLast));
        }
        for (WordNode subNode : subNodes) {
            if (subNode.value == value) {
                if (!subNode.isLast && isLast) {
                    subNode.isLast = true;
                }
                return subNode;
            }
        }
        return addSubNode(new WordNode(value, isLast));
    }

    public WordNode querySub(final int value) {
        if (subNodes == null) {
            return null;
        }
        for (WordNode subNode : subNodes) {
            if (subNode.value == value) {
                return subNode;
            }
        }
        return null;
    }

    public boolean isLast() {
        return isLast;
    }

    public void setLast(boolean isLast) {
        this.isLast = isLast;
    }

    @Override
    public int hashCode() {
        return value;
    }

}