aho/welcomecenterbot
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

spacy-words-separation2

Browse Source
This commit is contained in:
Untone
2024-09-28 12:06:24 +03:00
parent 56a2632980
commit 7030e58f4b
3 changed files with 8 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
nlp/segment_text.py
View File

@@ -1,7 +1,8 @@
import spacy
from spacy.lang.ru.examples import
# Load the Russian language model
nlp = spacy.load("ru_core_news_sm")
nlp = spacy.load("ru_core_news_md")
def segment_text(text):
"""
@@ -11,7 +12,7 @@ def segment_text(text):
doc = nlp(text)
# Extract words from the processed document
segmented_text = ' '.join([token.text for token in doc])
segmented_text = ' '.join([token.text for token in doc if not token.is_space])
return segmented_text