nlp（一）语种检测

布满荆棘的人生 2022-06-07 14:10 193阅读 0赞

# -*- coding: utf-8 -*-
    """
    Spyder Editor
    
    This is a temporary script file.
    """
    # NGram类计算字母序列出现的频率
    class NGram(object):
        # # text 一个unicode的文本输入
        # n作为定义字符序列的长度
    	def __init__(self, text, n=3):
    		self.length = None
    		self.n = n
    		self.table = {}
    		self.parse_text(text)
    		self.calculate_length()
        # 得到3－gram 的词典
    	def parse_text(self, text):
    		chars = ' ' * self.n # initial sequence of spaces with length n
    		for letter in (" ".join(text.split()) + " "):
    			chars = chars[1:] + letter # append letter to sequence of length n
    			self.table[chars] = self.table.get(chars, 0) + 1 # increment count
        # NGram对象的长度
    	def calculate_length(self):
    		""" Treat the N-Gram table as a vector and return its scalar magnitude
    		to be used for performing a vector-based search.
    		"""
    		self.length = sum([x * x for x in self.table.values()]) ** 0.5
    		return self.length
        # 两个NGram对象之间的减法
    	def __sub__(self, other):
    		""" Find the difference between two NGram objects by finding the cosine
    		of the angle between the two vector representations of the table of
    		N-Grams. Return a float value between 0 and 1 where 0 indicates that
    		the two NGrams are exactly the same.
    		"""
    		if not isinstance(other, NGram):
    			raise TypeError("Can't compare NGram with non-NGram object.")
     
    		if self.n != other.n:
    			raise TypeError("Can't compare NGram objects of different size.")
     
    		total = 0
    		for k in self.table:
    			total += self.table[k] * other.table.get(k, 0)
     
    		return 1.0 - (float(total) )/ (float(self.length) * float(other.length))
    
    
    if __name__=='__main__':
        training_text={'English':'training set EN.txt', 'German':'training set DE.txt', 'French':'training set FR.txt', 'Italian':'training set IT.txt'}
        encodingType = "UTF-8"
        readType = 'r'
        for n in range(1, 11):
            languages = {}
            filename = str(n) + '.txt'
            text_sfile = open(filename, readType, encoding=encodingType, errors='ignore')
            text_set = text_sfile.read()
            text_sfile.close()
            for key in training_text:
                training_file = open(training_text[key], 'r')
                training_set = training_file.read()
                training_file.close()
                languages[key] = NGram(training_set, n=3)-NGram(text_set, n=3)
            print(filename, "is", min(languages.items(), key=lambda x: x[1])[0])

1、实验结果

![Center][]

2、实验环境

Win8.1    Anaconda3.6.0 PyCharm2016.1.4

3、语料

四种语料，10个测试集

[Center]: /images/20220607/42a01edb82ba4957b129ae681a0f4542.png