پائتھون سکرپٹ: کارپس سے جملے علیحدہ کرنا

pip install urduhack

from urduhack.normalization import normalize
from urduhack.tokenization import sentence_tokenizer

def remove_unwanted_characters(text):
    unwanted_characters = "ٌّْۭؒؓؐۙۤۚۧٓx#%,?@[]_`{}~¦¨¯´¸¿؛؟٬٬‘’’’’¢£¤¥+<>±«»×÷§©®°µ¶…۝۝ۙ؎؀٭٭۞؏؍؁؃nanßþüýÿۗ$&*﴾.0123456789:=;¡ª²³¹º¼½¾ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞĄĆĈĊČĎǺǼàáâãäåæçèéêëìíîïðñòóôõöøùúû!"
    translation_table = str.maketrans('', '', unwanted_characters)
    cleaned_text = str(text).translate(translation_table)
    return cleaned_text


path = "d:\\sentences\\"
filename = "data.txt"

# read data
with open(path + filename, 'r', encoding="utf-8") as f:
    lines = f.readlines()

text = ""
for line in lines:
    text = text + line

#text normalization
normalized_text = normalize(text)

#sentence tokenization
sentences = sentence_tokenizer(normalized_text)

clean_sentences=[]
#remove unwanted characters
for s in sentences:
    clean_sentences.append(remove_unwanted_characters(s))

#remove duplicates
# clean_sentences = list(filter(None, clean_sentences ))
clean_sentences = list(set(clean_sentences))

# write sentence to file based on sentence length
for s in clean_sentences:
    s = s.replace("\n\n","")
    s = s.replace("\n","")
    filename = path + str(len(s.split())).zfill(3) +".txt"
    with open(filename, 'a', encoding="utf-8") as f:
        f.write(s+"\n")

print("done...")

path = "d:\\sentences\\"

filename = "data.txt"