aho/welcomecenterbot
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

average-stabw

Browse Source
This commit is contained in:
Untone 2024-09-27 13:39:40 +03:00
parent 773a01cdb1
commit c9728e14a7
2 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
handlers/messages_routing.py
View File

@ -92,7 +92,7 @@ async def messages_routing(msg, state):
except Exception:
pass
else:
toxic_score = detector(text)
toxic_score = detector(normalize(text))
toxic_perc = math.floor(toxic_score * 100)
await redis.set(f"toxic:{cid}", mid)
await redis.set(f"toxic:{cid}:{uid}:{mid}", toxic_perc, ex=60 * 60 * 24 * 3)

4
nlp/normalize.py
View File

@ -44,8 +44,10 @@ def normalize(text):
"""
Normalize English text to resemble Russian characters.
"""
t = text.replace(" ", " ").replace(" ", " ").replace(" ", " ")
# Segment the text first
t = segment_text(text.replace(" ", " ").replace(" ", " ").replace(" ", " "))
# t = segment_text(t)
t = t.lower()