Python优化

Question

f = open('wl4.txt', 'w')
hh = 0
######################################
for n in range(1,5):
    for l in range(33,127):
        if n==1:
            b = chr(l) + '\n'
            f.write(b)
            hh += 1 
        elif n==2:           
            for s0 in range(33, 127):
                b = chr(l) + chr(s0) + '\n'
                f.write(b)
                hh += 1
        elif n==3:          
            for s0 in range(33, 127):
                for s1 in range(33, 127):
                    b = chr(l) + chr(s0) + chr(s1) + '\n'
                    f.write(b)
                    hh += 1 
        elif n==4:    
            for s0 in range(33, 127):
                for s1 in range(33, 127):
                    for s2 in range(33,127):
            b = chr(l) + chr(s0) + chr(s1) + chr(s2) + '\n'
            f.write(b)
            hh += 1 
######################################
print "We Made %d Words." %(hh)
######################################
f.close()

那么，有没有什么方法可以让它更快呢？

Answer 1

进一步的重大改进是可能的。

以下脚本文件演示了这些，(为简洁起见)仅使用大小为 4 的循环(它占用了 90% 以上的时间)。

方法0:OP的原始代码

方法一:John Kugleman 的解决方案

方法 2:(1) 并将一些字符串连接移出内部循环

方法 3:(2) 并将代码放在一个函数中——访问局部变量比全局变量快得多。任何脚本都可以做到这一点。许多脚本应该这样做。

方法4:(3)将字符串累积到一个列表中，然后将它们连接起来并写入。请注意，这会使用您可能不相信的内存。我的代码不会尝试对整个文件执行此操作，因为 (127 - 33) ** 4 是 78M 字符串。在 32 位机器上，仅列表就有 78 * 4 = 312Mb(忽略列表末尾未使用的内存)，加上 str 对象的 78 * 28 = 2184 Mb(sys.getsizeof("1234") 产生28)，加上 78 * 5 = 390 Mb 作为连接结果。您只是破坏了您的用户地址空间或您的 ulimit 或其他可破坏的东西。或者，如果您有 1 Gb 的实际内存，其中 128 Mb 已被视频驱动程序占用，但有足够的交换空间，您有时间吃午餐(如果运行特定操作系统，还有晚餐)。

方法 5:(4)并且不要向列表询问其追加属性的下落 7800 万次:-)

这是脚本文件:

import time, sys
time_function = time.clock # Windows; time.time may be better on *x
ubound, which = map(int, sys.argv[1:3])
t0 = time_function()
if which == 0:
    ### original ###
    f = open('wl4.txt', 'w')
    hh = 0
    n = 4
    for l in range(33, ubound):
        if n == 1:
            pass
        elif n == 2:
            pass
        elif n == 3:
            pass
        elif n == 4:
            for s0 in range(33, ubound):
                for s1 in range(33, ubound):
                    for s2 in range(33,ubound):
                        b = chr(l) + chr(s0) + chr(s1) + chr(s2) + '\n'
                        f.write(b)
                        hh += 1
    f.close()
elif which == 1:
    ### John Kugleman ###
    f = open('wl4.txt', 'w')
    chars = [chr(c) for c in range(33, ubound)]
    hh = 0
    for l in chars:
        for s0 in chars:
            for s1 in chars:
                for s2 in chars:
                    b = l + s0 + s1 + s2 + '\n'
                    f.write(b)
                    hh += 1
    f.close()
elif which == 2:
    ### JohnK, saving + ###
    f = open('wl4.txt', 'w')
    chars = [chr(c) for c in range(33, ubound)]
    hh = 0
    for L in chars: # "L" as in "Legible" ;-)
        for s0 in chars:
            b0 = L + s0
            for s1 in chars:
                b1 = b0 + s1
                for s2 in chars:
                    b = b1 + s2 + '\n'
                    f.write(b)
                    hh += 1
    f.close()
elif which == 3:
    ### JohnK,  saving +, function ###
    def which3func():
        f = open('wl4.txt', 'w')
        chars = [chr(c) for c in range(33, ubound)]
        nwords = 0
        for L in chars:
            for s0 in chars:
                b0 = L + s0
                for s1 in chars:
                    b1 = b0 + s1
                    for s2 in chars:
                        b = b1 + s2 + '\n'
                        f.write(b)
                        nwords += 1
        f.close()
        return nwords
    hh = which3func()
elif which == 4:
    ### JohnK, saving +, function, linesep.join() ###
    def which4func():
        f = open('wl4.txt', 'w')
        chars = [chr(c) for c in range(33, ubound)]
        nwords = 0
        for L in chars:
            accum = []
            for s0 in chars:
                b0 = L + s0
                for s1 in chars:
                    b1 = b0 + s1
                    for s2 in chars:
                        accum.append(b1 + s2)
            nwords += len(accum)
            accum.append("") # so that we get a final newline
            f.write('\n'.join(accum))
        f.close()
        return nwords
    hh = which4func()
elif which == 5:
    ### JohnK, saving +, function, linesep.join(), avoid method lookup in loop ###
    def which5func():
        f = open('wl4.txt', 'w')
        chars = [chr(c) for c in range(33, ubound)]
        nwords = 0
        for L in chars:
            accum = []; accum_append = accum.append
            for s0 in chars:
                b0 = L + s0
                for s1 in chars:
                    b1 = b0 + s1
                    for s2 in chars:
                        accum_append(b1 + s2)
            nwords += len(accum)
            accum_append("") # so that we get a final newline
            f.write('\n'.join(accum))
        f.close()
        return nwords
    hh = which5func()
else:
    print "Bzzzzzzt!!!"
t1 = time_function()
print "Method %d made %d words in %.1f seconds" % (which, hh, t1 - t0)

下面是一些结果:

C:\junk\so>for %w in (0 1 2 3 4 5) do \python26\python wl4.py 127 %w

C:\junk\so>\python26\python wl4.py 127 0
Method 0 made 78074896 words in 352.3 seconds

C:\junk\so>\python26\python wl4.py 127 1
Method 1 made 78074896 words in 183.9 seconds

C:\junk\so>\python26\python wl4.py 127 2
Method 2 made 78074896 words in 157.9 seconds

C:\junk\so>\python26\python wl4.py 127 3
Method 3 made 78074896 words in 126.0 seconds

C:\junk\so>\python26\python wl4.py 127 4
Method 4 made 78074896 words in 68.3 seconds

C:\junk\so>\python26\python wl4.py 127 5
Method 5 made 78074896 words in 60.5 seconds

更新以回应 OP 的问题

"""当我尝试添加 for 循环时，我收到 accum_append 的内存错误。问题是什么？？"""

我不知道问题是什么；我无法在这个距离读取您的代码。猜测:如果你试图做 length == 5，你可能得到了 accum 初始化并在错误的地方写入位，并且 accum 正试图超出系统内存的容量(我希望我之前已经解释过了)。

"""现在方法 5 是最快的方法，但它使一个字告诉长度 4.. 我怎么能得到我想要的多少？？:)"""

你有两个选择:(1)你继续使用嵌套 for 循环(2)你查看不使用嵌套 for 循环的答案，其长度是动态指定的。

方法 4 和 5 通过使用 accum 获得了加速，但这样做的方式是根据对将使用多少内存的确切知识量身定制的。

下面是另外 3 种方法。 101 是 tgray 的方法，没有额外的内存使用。 201 是 Paul Hankin 的方法(加上一些写入文件的代码)，同样没有额外的内存使用。这两种方法速度差不多，都在方法3 speedwise 的视线之内。它们都允许动态指定所需的长度。

方法 102 是 tgray 的方法，具有固定的 1Mb 缓冲区——它试图通过减少对 f.write() 的调用次数来节省时间……您可能希望尝试缓冲区大小。如果愿意，您可以创建正交 202 方法。请注意，tgray 的方法使用 itertools.product，您需要 Python 2.6，而 Paul Hankin 的方法使用已经存在一段时间的生成器表达式。

elif which == 101:
    ### tgray, memory-lite version
    def which101func():
        f = open('wl4.txt', 'w')
        f_write = f.write
        nwords = 0
        chars = map(chr, xrange(33, ubound))  # create a list of characters
        length = 4 #### length is a variable
        for x in product(chars, repeat=length):
            f_write(''.join(x) + '\n')
            nwords += 1
        f.close()
        return nwords
    hh = which101func()
elif which == 102:
    ### tgray, memory-lite version, buffered
    def which102func():
        f = open('wl4.txt', 'w')
        f_write = f.write
        nwords = 0
        chars = map(chr, xrange(33, ubound))  # create a list of characters
        length = 4 #### length is a variable
        buffer_size_bytes = 1024 * 1024
        buffer_size_words = buffer_size_bytes // (length + 1)
        words_in_buffer = 0
        buffer = []; buffer_append = buffer.append
        for x in product(chars, repeat=length):
            words_in_buffer += 1
            buffer_append(''.join(x) + '\n')
            if words_in_buffer >= buffer_size_words:
                f_write(''.join(buffer))
                nwords += words_in_buffer
                words_in_buffer = 0
                del buffer[:]
        if buffer:
            f_write(''.join(buffer))
            nwords += words_in_buffer
        f.close()
        return nwords
    hh = which102func()
elif which == 201:
    ### Paul Hankin (needed output-to-file code added)
    def AllWords(n, CHARS=[chr(i) for i in xrange(33, ubound)]):
        #### n is the required word length
        if n == 1: return CHARS
        return (w + c for w in AllWords(n - 1) for c in CHARS)
    def which201func():
        f = open('wl4.txt', 'w')
        f_write = f.write
        nwords = 0
        for w in AllWords(4):
            f_write(w + '\n')
            nwords += 1
        f.close()
        return nwords
    hh = which201func()