乙個簡單的實現
class
*****filter()
:'''filter messages from keywords
very ****** filter implementation
>>> f = *****filter()
>>> f.add("***y")
>>> f.filter("hello ***y baby")
hello **** baby
'''def__init__
(self)
: self.keywords =
set(
)def
parse
(self, path)
:for keyword in
open
(path)
: self.keywords.add(keyword.strip(
).decode(
'utf-8'
).lower())
deffilter
(self, message, repl=
"*")
: message =
str(message)
.lower(
)for kw in self.keywords:
message = message.replace(kw, repl)
return message
class
bsfilter
:'''filter messages from keywords
>>> f = bsfilter()
>>> f.add("***y")
>>> f.filter("hello ***y baby")
hello **** baby
'''def__init__
(self)
: self.keywords =
self.kwsets =
set(
) self.bsdict = defaultdict(
set)
self.pat_en = re.
compile
(r'^[0-9a-za-z]+$'
)# english phrase or not
defadd
(self, keyword):if
notisinstance
(keyword,
str)
: keyword = keyword.decode(
'utf-8'
) keyword = keyword.lower(
)if keyword not
in self.kwsets:
self.kwsets.add(keyword)
index =
len(self.keywords)-1
for word in keyword.split():
if self.pat_en.search(word)
: self.bsdict[word]
.add(index)
else
:for char in word:
self.bsdict[char]
.add(index)
defparse
(self, path)
:with
open
(path,
"r")
as f:
for keyword in f:
self.add(keyword.strip())
deffilter
(self, message, repl=
"*"):if
notisinstance
(message,
str)
: message = message.decode(
'utf-8'
) message = message.lower(
)for word in message.split():
if self.pat_en.search(word)
:for index in self.bsdict[word]
: message = message.replace(self.keywords[index]
, repl)
else
:for char in word:
for index in self.bsdict[char]
: message = message.replace(self.keywords[index]
, repl)
return message
bfs：寬度優先搜尋方式。
class
dfafilter()
:'''filter messages from keywords
use dfa to keep algorithm perform constantly
>>> f = dfafilter()
>>> f.add("***y")
>>> f.filter("hello ***y baby")
hello **** baby
'''def__init__
(self)
: self.keyword_chains =
self.delimit =
'\x00'
defadd
(self, keyword):if
notisinstance
(keyword,
str)
: keyword = keyword.decode(
'utf-8'
) keyword = keyword.lower(
) chars = keyword.strip()if
not chars:
return
level = self.keyword_chains
for i in
range
(len
(chars)):
if chars[i]
in level:
level = level[chars[i]
]else:if
notisinstance
(level,
dict):
break
for j in
range
(i,len
(chars)):
level[chars[j]]=
last_level, last_char = level, chars[j]
level = level[chars[j]
] last_level[last_char]
=break
if i ==
len(chars)-1
: level[self.delimit]=0
defparse
(self, path)
:with
open
(path,encoding=
'utf-8'
)as f:
for keyword in f:
self.add(keyword.strip())
deffilter
(self, message, repl=
"*"):if
notisinstance
(message,
str)
: message = message.decode(
'utf-8'
) message = message.lower(
) ret =
start =
0while start <
len(message)
: level = self.keyword_chains
step_ins =
0for char in message[start:]:
if char in level:
step_ins +=
1if self.delimit not
in level[char]
: level = level[char]
else
: start += step_ins -
1break
else:)
break
else:)
start +=
1return
''.join(ret)
使用了巢狀的字典來實現。
敏感詞過濾演算法實現
說到敏感詞過濾，我也覺得這裡沒有必要寫這個文章，因為前人已經前前後後有過很多種演算法解決該問題。這裡我之所以寫這個文章，是因為我自己自創了一種演算法 真的是自創哦，因為我在寫這個演算法的時候，完全是自己想出來的方式，沒有借鑑任何 靈感來自於一篇文章中的一句話 如果能掃瞄一遍文字就能將所有的詞找出來，...
PHP實現敏感詞過濾
為了構建良好的網路環境以及為了大家健康的上網。編不下去了 反正你知道敏感詞過濾一定是要有的，如果沒有這東西有關部門不找你喝茶哦！壞笑 壞笑 廢話不多說指尖上乾貨 fake file get contents words.txt 讀取關鍵字文字資訊 這個文字就是敏感詞的文字網上都有 最好選擇最新最全的...
spring boot 實現敏感詞過濾
首先新建乙個敏感詞過濾工具類，目的是傳入乙個字串，可以自動過濾並返回。如下 敏感詞處理工具 dfa演算法實現 author 2019 shiwq public class sensitivewordutil 初始化敏感詞庫，構建dfa演算法模型 param sensitivewordset 敏感詞庫...