我尝试将NLTK工具包中存在的QuadgramCollationFinde类转换为PentagramCollocationFinder one，即使用5个项目，而不是仅使用4个项目，但确实有效。
这是 nltk 类
import nltk
from nltk.compat import iteritems
from nltk.probability import FreqDist
from nltk.util import ngrams
from nltk.metrics import ContingencyMeasures, BigramAssocMeasures,TrigramAssocMeasures
from nltk.metrics.spearman import ranks_from_scores, spearman_correlation
from __future__ import print_function
class QuadgramCollocationFinder(AbstractCollocationFinder):
"""A tool for the finding and ranking of quadgram collocations or other association measures.
It is often useful to use from_words() rather than constructing an instance directly.
"""
default_ws = 4
def __init__(self, word_fd, quadgram_fd, ii, iii, ixi, ixxi, iixi, ixii):
"""Construct a QuadgramCollocationFinder, given FreqDists for appearances of words,
bigrams, trigrams, two words with one word and two words between them, three words
with a word between them in both variations.
"""
AbstractCollocationFinder.__init__(self, word_fd, quadgram_fd)
self.iii = iii
self.ii = ii
self.ixi = ixi
self.ixxi = ixxi
self.iixi = iixi
self.ixii = ixii
def from_words(cls, words, window_size=4):
if window_size < 4:
raise ValueError("Specify window_size at least 4")
ixxx = FreqDist()
iiii = FreqDist()
ii = FreqDist()
iii = FreqDist()
ixi = FreqDist()
ixxi = FreqDist()
iixi = FreqDist()
ixii = FreqDist()
for window in ngrams(words, window_size, pad_right=True):
w1 = window[0]
if w1 is None:
continue
for w2, w3, w4 in _itertools.combinations(window[1:], 3):
ixxx[w1] += 1
if w2 is None:
continue
ii[(w1, w2)] += 1
if w3 is None:
continue
iii[(w1, w2, w3)] += 1
ixi[(w1, w3)] += 1
if w4 is None:
continue
iiii[(w1, w2, w3, w4)] += 1
ixxi[(w1, w4)] += 1
ixii[(w1, w3, w4)] += 1
iixi[(w1, w2, w4)] += 1
return cls(ixxx, iiii, ii, iii, ixi, ixxi, iixi, ixii)
def score_ngram(self, score_fn, w1, w2, w3, w4):
n_all = self.word_fd.N()
n_iiii = self.ngram_fd[(w1, w2, w3, w4)]
if not n_iiii:
return
n_iiix = self.iii[(w1, w2, w3)]
n_xiii = self.iii[(w2, w3, w4)]
n_iixi = self.iixi[(w1, w2, w4)]
n_ixii = self.ixii[(w1, w3, w4)]
n_iixx = self.ii[(w1, w2)]
n_xxii = self.ii[(w3, w4)]
n_xiix = self.ii[(w2, w3)]
n_ixix = self.ixi[(w1, w3)]
n_ixxi = self.ixxi[(w1, w4)]
n_xixi = self.ixi[(w2, w4)]
n_ixxx = self.word_fd[w1]
n_xixx = self.word_fd[w2]
n_xxix = self.word_fd[w3]
n_xxxi = self.word_fd[w4]
return score_fn(n_iiii,
(n_iiix, n_iixi, n_ixii, n_xiii),
(n_iixx, n_ixix, n_ixxi, n_xixi, n_xxii, n_xiix),
(n_ixxx, n_xixx, n_xxix, n_xxxi),
n_all)
这就是我想要的
class fivegramCollocationFinder(AbstractCollocationFinder):
"""A tool for the finding and ranking of quadgram collocations or other association measures.
It is often useful to use from_words() rather than constructing an instance directly.
"""
default_ws = 4
def __init__(self, word_fd, quingram_fd, ii, iii, ixi,iiii,ixxi, ixii, iixi, ixxxi, iixxi, ixixi, ixxii, iiixi, ixiii, iixii):
"""Construct a QuadgramCollocationFinder, given FreqDists for appearances of words,
bigrams, trigrams, two words with one word and two words between them, three words
with a word between them in both variations.
"""
AbstractCollocationFinder.__init__(self, word_fd, quingram_fd)
self.iiii = iiii
self.iii = iii
self.ii = ii
self.ixi = ixi
self.ixxi = ixxi
self.iixi = iixi
self.ixii = ixii
self.ixxxi = ixxxi
self.iixxi = iixxi
self.ixixi = ixixi
self.ixxii = ixxii
self.iiixi = iiixi
self.ixiii = ixiii
self.iixii = iixii
@classmethod
def from_words(cls, words, window_size=5):
if window_size < 5:
raise ValueError("Specify window_size at least 4")
ixxxx = FreqDist()
ii = FreqDist()
iii = FreqDist()
iiii = FreqDist()
iiiii = FreqDist()
ixi = FreqDist()
ixxi = FreqDist()
iixi = FreqDist()
ixii = FreqDist()
ixxxi = FreqDist()
iixxi = FreqDist()
ixixi = FreqDist()
ixxii = FreqDist()
iiixi = FreqDist()
ixiii = FreqDist()
iixii = FreqDist()
for window in ngrams(words, window_size, pad_right=True):
w1 = window[0]
if w1 is None:
continue
for w2, w3, w4, w5 in _itertools.combinations(window[1:], 4):
ixxxx[w1] += 1
if w2 is None:
continue
ii[(w1, w2)] += 1
if w3 is None:
continue
iii[(w1, w2, w3)] += 1
ixi[(w1, w3)] += 1
if w4 is None:
continue
iiii[(w1, w2, w3, w4)] += 1
ixxi[(w1, w4)] += 1
ixii[(w1, w3, w4)] += 1
iixi[(w1, w2, w4)] += 1
if w5 is None:
continue
iiiii[(w1, w2, w3, w4, w5)] += 1
ixxxi[(w1, w5)] += 1
iixxi[(w1, w2, w5)] += 1
ixixi[(w1, w3, w5)] += 1
ixxii[(w1, w4, w5)] += 1
iiixi[(w1, w2, w3, w5)] +=1
ixiii[(w1, w3, w4, w5)] +=1
iixii[(w1, w2, w4, w5)] +=1
return cls(ixxxx, ii,iii, ixi, ixxi, iixi, ixii,iiii, iiiii, ixxxi, iixxi, ixixi, ixxii, iiixi, ixiii, iixii)
def score_ngram(self, score_fn, w1, w2, w3, w4, w5):
n_all = self.word_fd.N()
n_iiiii = self.ngram_fd[(w1, w2, w3, w4, w5)]
if not n_iiiii:
return
n_iiixi = self.iiixi[(w1, w2, w3, w5)]
n_ixiii = self.ixiii[(w1, w3, w4, w5)]
n_iixii = self.iixii[(w1, w2, w4, w5)]
n_iiiix = self.iiii[(w1,w2,w3,w4)]
n_xiiii = self.iiii[(w2,w3,w4,w5)]
n_iixix = self.iixix[(w1, w2, w4)]
n_iixxi = self.iixxi[(w1, w2, w5)]
n_ixixi = self.ixixi[(w1, w3, w5)]
n_ixxii = self.ixxii[(w1, w4, w5)]
n_xiixi = self.xiixi[(w2, w3, w5)]
n_xixii = self.xixii[(w2, w4, w5)]
n_ixiix = self.ixiix[(w1, w3, w4)]
n_iiixx = self.iii[(w1, w2, w3)]
n_xiiix = self.iii[(w2, w3, w4)]
n_xxiii = self.iii[(w3, w4, w5)]
n_ixixx = self.ixixx[(w1, w3)]
n_ixxix = self.ixxix[(w1, w4)]
n_ixxxi = self.ixxxi[(w1, w5)]
n_xixix = self.xixix[(w2, w4)]
n_xixxi = self.xixxi[(w2, w5)]
n_xxixi = self.xxixi[(w2, w5)]
n_iixxx = self.ii[(w1, w2)]
n_xiixx = self.ii[(w2, w3)]
n_xxiix= self.ii[(w3, w4)]
n_xxxii= self.ii[(w4, w5)]
n_ixxxx = self.word_fd[w1]
n_xixxx = self.word_fd[w2]
n_xxixx = self.word_fd[w3]
n_xxxix = self.word_fd[w4]
n_xxxxi = self.word_fd[w5]
return score_fn(n_iiiii,
(n_iiixi, n_ixiii, n_iixii, n_iiiix, n_xiiii),
(n_iixix,n_iixxi, n_ixixi, n_ixxii, n_xiixi, n_xixii,n_ixiix, n_iiixx, n_xiiix, n_xxiii),
(n_ixixx, n_ixxix,n_ixxxi, n_xixix, n_xixxi, n_xxixi, n_iixxx, n_xiixx,n_xxiix,n_xxxii),
(n_ixxxx,n_xixxx, n_xxixx, n_xxxix, n_xxxxi),
n_all)
所以需要更新类来检测PentagramCollocationFinder
有人可以帮忙吗？
最佳答案
构建模式似乎有些令人担忧，因此这里有一些构建所有合法 i 模式和要使用的 n 模式的代码。
import collections
def make_ngram_ipatterns(n):
"""Make all needed patterns used by *gramCollocationFinder up to n words"""
i_patterns = []
for i in xrange(1, n+1):
if i <= 2:
i_patterns.append('i' * i)
else:
for j in xrange(2**(i-2)):
bin_str = '{0:0{1}b}'.format(j, i-2)
ix_pattern = bin_str.replace('0', 'x').replace('1', 'i')
i_patterns.append('i{}i'.format(ix_pattern))
return i_patterns
def make_ngram_npatterns(n):
"""Make all needed n-patterings used by *gramCollocationFinder up to n words"""
all_ipatterns = make_ngram_ipatterns(n)
npatterns = []
for ipattern in all_ipatterns:
i_order = sum(c == 'i' for c in ipattern)
i_length = len(ipattern)
for j in xrange(n - i_length+1):
npattern = 'n_{}{}{}'.format('x'* j,
ipattern ,
'x'* (n - i_length - j))
npatterns.append((i_order, ipattern, npattern))
return sorted(npatterns)
def main():
n = 5
all_ipatterns = make_ngram_ipatterns(n)
print '\n'.join(make_ngram_ipatterns(n))
for order, ipattern, npattern in make_ngram_npatterns(n):
wparams = ', '.join('w{}'.format(i+1)
for i, c in enumerate(npattern[2:])
if c == 'i'
)
print('order: {1:2} ipattern: {2:{0}s} npattern: {3}'
' -> {3} = self.{2}({4})'.format(
n, order, ipattern, npattern, wparams))
if __name__ == '__main__':
main()
n=5 的输出如下:
i
ii
ixi
iii
ixxi
ixii
iixi
iiii
ixxxi
ixxii
ixixi
ixiii
iixxi
iixii
iiixi
iiiii
order: 1 ipattern: i npattern: n_ixxxx -> n_ixxxx = self.i(w1)
order: 1 ipattern: i npattern: n_xixxx -> n_xixxx = self.i(w2)
order: 1 ipattern: i npattern: n_xxixx -> n_xxixx = self.i(w3)
order: 1 ipattern: i npattern: n_xxxix -> n_xxxix = self.i(w4)
order: 1 ipattern: i npattern: n_xxxxi -> n_xxxxi = self.i(w5)
order: 2 ipattern: ii npattern: n_iixxx -> n_iixxx = self.ii(w1, w2)
order: 2 ipattern: ii npattern: n_xiixx -> n_xiixx = self.ii(w2, w3)
order: 2 ipattern: ii npattern: n_xxiix -> n_xxiix = self.ii(w3, w4)
order: 2 ipattern: ii npattern: n_xxxii -> n_xxxii = self.ii(w4, w5)
order: 2 ipattern: ixi npattern: n_ixixx -> n_ixixx = self.ixi(w1, w3)
order: 2 ipattern: ixi npattern: n_xixix -> n_xixix = self.ixi(w2, w4)
order: 2 ipattern: ixi npattern: n_xxixi -> n_xxixi = self.ixi(w3, w5)
order: 2 ipattern: ixxi npattern: n_ixxix -> n_ixxix = self.ixxi(w1, w4)
order: 2 ipattern: ixxi npattern: n_xixxi -> n_xixxi = self.ixxi(w2, w5)
order: 2 ipattern: ixxxi npattern: n_ixxxi -> n_ixxxi = self.ixxxi(w1, w5)
order: 3 ipattern: iii npattern: n_iiixx -> n_iiixx = self.iii(w1, w2, w3)
order: 3 ipattern: iii npattern: n_xiiix -> n_xiiix = self.iii(w2, w3, w4)
order: 3 ipattern: iii npattern: n_xxiii -> n_xxiii = self.iii(w3, w4, w5)
order: 3 ipattern: iixi npattern: n_iixix -> n_iixix = self.iixi(w1, w2, w4)
order: 3 ipattern: iixi npattern: n_xiixi -> n_xiixi = self.iixi(w2, w3, w5)
order: 3 ipattern: iixxi npattern: n_iixxi -> n_iixxi = self.iixxi(w1, w2, w5)
order: 3 ipattern: ixii npattern: n_ixiix -> n_ixiix = self.ixii(w1, w3, w4)
order: 3 ipattern: ixii npattern: n_xixii -> n_xixii = self.ixii(w2, w4, w5)
order: 3 ipattern: ixixi npattern: n_ixixi -> n_ixixi = self.ixixi(w1, w3, w5)
order: 3 ipattern: ixxii npattern: n_ixxii -> n_ixxii = self.ixxii(w1, w4, w5)
order: 4 ipattern: iiii npattern: n_iiiix -> n_iiiix = self.iiii(w1, w2, w3, w4)
order: 4 ipattern: iiii npattern: n_xiiii -> n_xiiii = self.iiii(w2, w3, w4, w5)
order: 4 ipattern: iiixi npattern: n_iiixi -> n_iiixi = self.iiixi(w1, w2, w3, w5)
order: 4 ipattern: iixii npattern: n_iixii -> n_iixii = self.iixii(w1, w2, w4, w5)
order: 4 ipattern: ixiii npattern: n_ixiii -> n_ixiii = self.ixiii(w1, w3, w4, w5)
order: 5 ipattern: iiiii npattern: n_iiiii -> n_iiiii = self.iiiii(w1, w2, w3, w4, w5)
更改为新维度现在只需使用所有 i 模式并将其设置为低阶类，替换 n 模式，并将所有相同阶的 n 模式整理到 score_fn()集。
编辑:使用适当的 w# 完成 n 模式的设置
关于python - 将 QuadgramCollationFinder 转换为 PentagramCollationFinder，我们在Stack Overflow上找到一个类似的问题： https://stackoverflow.com/questions/33021916/