aho/welcomecenterbot
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added-fix

Browse Source
This commit is contained in:
Untone 2024-09-29 14:12:55 +03:00
parent 19a93d2443
commit 9c0d3cf4e1
2 changed files with 11 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
handlers/messages_routing.py
View File

@ -112,14 +112,18 @@ async def messages_routing(msg, state):
text += '\n' text += '\n'
normalized_text = normalize(text) normalized_text = normalize(text)
logger.info(f"normalized text: {normalized_text}")
toxic_score = detector(normalized_text) toxic_score = detector(normalized_text)
sw_score = 0
if toxic_score < 0.91:
logger.info('re-check stopwords in combinations')
stopwords_detected = check_stopwords(normalized_text)
for stopword in stopwords_detected:
sw_score += detect(stopword)
toxic_perc = toxic_score * 100 toxic_perc = toxic_score * 100
logger.info(f'original toxic: {toxic_perc}') logger.info(f'original toxic: {toxic_perc}')
if toxic_score < 0.91: toxic_perc = (toxic_score + sw_score) * 100
logger.info('re-check without spaces') logger.info(f'added stopwords toxic: {sw_score*100}')
toxic_perc += check_stopwords(normalized_text)
logger.info(f"text: {normalized_text}\ntoxic: {toxic_perc}%")
await redis.set(f"toxic:{cid}", mid) await redis.set(f"toxic:{cid}", mid)
await redis.set(f"toxic:{cid}:{uid}:{mid}", math.floor(toxic_perc), ex=60 * 60 * 24 * 3) await redis.set(f"toxic:{cid}:{uid}:{mid}", math.floor(toxic_perc), ex=60 * 60 * 24 * 3)
if toxic_score > 0.75: if toxic_score > 0.75:

16
nlp/stopwords_detector.py
View File

@ -72,16 +72,7 @@ def check_stopwords(text):
Examples: Examples:
>>> check_stopwords("this is a хуй") >>> check_stopwords("this is a хуй")
40 {'хуй'}
>>> check_stopwords("this is clean")
0
>>> check_stopwords("хуй is a хуй")
80
>>> check_stopwords("clean is clean")
0
""" """
# Normalize the text by splitting into words # Normalize the text by splitting into words
@ -90,10 +81,7 @@ def check_stopwords(text):
# Check for any intersection with stopword_set # Check for any intersection with stopword_set
stopwords_found = stopword_set.intersection(words) stopwords_found = stopword_set.intersection(words)
# Calculate the score based on the number of stopwords found return stopwords_found
score = 90 + len(stopwords_found)
return score
# Example usage # Example usage
if __name__ == "__main__": if __name__ == "__main__":