python - 在 Python 中并行化 for 循环以加快算法速度

Question

该算法包括读取以 Clipped.tiff 结尾的文件夹中的所有图像，并且要加速更改所有扫描图像的 Gamma 值的 for 循环。

如何并行化并加速这个简单的算法？

SinglestreetsFolder = "/dop_shapefile/"
shp_list = [x for x in os.listdir(SinglestreetsFolder) if x.endswith("clipped.tiff")]


for i in range(0, len(shp_list), 1):

    originalPath = "/dop_shapefile/" + shp_list[i]
    original = cv2.imread(originalPath)

    adjusted = adjust_gamma(original, gamma=0.3)

    cv2.imwrite("/gamma/" + shp_list[i] + "_gamma.tiff", adjusted)
    print "status_Gamma: ", i + 1, "/", len(shp_list)

---

另一种算法

该算法包括读取以 .tiff 结尾的文件夹中的所有图像和 for 循环，该循环执行 ConnectedComponentLabeling 算法(可自定义)以加快速度。

SinglestreetsFolder = "/gabor/"
shp_list = [x for x in os.listdir(SinglestreetsFolder) if x.endswith(".tiff")]


img = cv2.imread(SinglestreetsFolder + shp_list[0], 0)
wholeimage = np.zeros(shape=(len(img), len(img[0]), 3))
erosion = np.ones((2, 2), np.uint8)     # kernel: erosion

for j in range(0, len(shp_list), 1):
    if "motorway" in shp_list[j]:
        N = 40  # pixel threshold (city: ca. 10, motorway: ca. 40)
    else:
        N = 10   # pixel threshold (city: ca. 10, motorway: ca. 40)

    img = cv2.imread(SinglestreetsFolder + shp_list[j], 0)
    connectivity = 8    # 4- OR 8-connectivity connected component labeling


    if "reverse" in shp_list[j]:
        img = cv2.threshold(img, 140, 255, cv2.THRESH_BINARY)[1]  # ensure binary

        img = cv2.morphologyEx(img, cv2.MORPH_OPEN, erosion)

        retval, labels = cv2.connectedComponents(img, connectivity)

        num = labels.max()

        # If the count of pixels less than a threshold, then set pixels to `0` (background)
        for i in range(1, num + 1):
            pts = np.where(labels == i)
            if len(pts[0]) < N:
                labels[pts] = 0

        # Map component labels to hue val
        label_hue = np.uint8(179 * labels / np.max(labels))
        blank_ch = 255 * np.ones_like(labels)
        labeled_img = cv2.merge([blank_ch, blank_ch, blank_ch])

        # set bg label to black
        labeled_img[label_hue == 0] = 0

        wholeimage = np.where(labeled_img == 0, wholeimage, labeled_img)

    else:
        img = cv2.threshold(img, 140, 255, cv2.THRESH_BINARY)[1]  # ensure binary

        retval, labels = cv2.connectedComponents(img, connectivity)

        num = labels.max()

        # If the count of pixels less than a threshold, then set pixels to `0` (background)
        for i in range(1, num + 1):
            pts = np.where(labels == i)
            if len(pts[0]) < N:
                labels[pts] = 0

        # Map component labels to hue val
        label_hue = np.uint8(179 * labels / np.max(labels))
        blank_ch = 255 * np.ones_like(labels)
        labeled_img = cv2.merge([blank_ch, blank_ch, blank_ch])

        # set bg label to black
        labeled_img[label_hue == 0] = 0

        wholeimage = np.where(labeled_img == 0, wholeimage, labeled_img)


    print "status_CCL: ", j + 1, "/", len(shp_list)

cv2.imwrite("/ccl/streets_gabor_ccl.tiff", wholeimage)

---

我的解决方案，但“整个图像”最后是黑色的

SinglestreetsFolder = "/gabor/"
shp_list = [x for x in os.listdir(SinglestreetsFolder) if x.endswith(".tiff")]


def gabor(params):
    img = cv2.imread(SinglestreetsFolder + shp_list[0], 0)
    wholeimage = np.zeros(shape=(len(img), len(img[0]), 3))
    erosion = np.ones((2, 2), np.uint8)  # kernel: erosion

    j, image_name = params

    if "motorway" in image_name:
        N = 40  # pixel threshold (city: ca. 10, motorway: ca. 40)
    else:
        N = 10   # pixel threshold (city: ca. 10, motorway: ca. 40)

    img = cv2.imread(SinglestreetsFolder + image_name, 0)
    connectivity = 8    # 4- OR 8-connectivity connected component labeling


    if "reverse" in image_name:
        img = cv2.threshold(img, 140, 255, cv2.THRESH_BINARY)[1]  # ensure binary

        img = cv2.morphologyEx(img, cv2.MORPH_OPEN, erosion)
        # img = cv2.erode(img, erosion, iterations=1)

        retval, labels = cv2.connectedComponents(img, connectivity)

        num = labels.max()

        # If the count of pixels less than a threshold, then set pixels to `0` (background)
        for i in range(1, num + 1):
            pts = np.where(labels == i)
            if len(pts[0]) < N:
                labels[pts] = 0

        # Map component labels to hue val
        label_hue = np.uint8(179 * labels / np.max(labels))
        blank_ch = 255 * np.ones_like(labels)
        labeled_img = cv2.merge([blank_ch, blank_ch, blank_ch])

        # set bg label to black
        labeled_img[label_hue == 0] = 0

        wholeimage = np.where(labeled_img == 0, wholeimage, labeled_img)

    else:
        img = cv2.threshold(img, 140, 255, cv2.THRESH_BINARY)[1]  # ensure binary

        retval, labels = cv2.connectedComponents(img, connectivity)

        num = labels.max()

        # If the count of pixels less than a threshold, then set pixels to `0` (background)
        for i in range(1, num + 1):
            pts = np.where(labels == i)
            if len(pts[0]) < N:
                labels[pts] = 0

        # Map component labels to hue val
        label_hue = np.uint8(179 * labels / np.max(labels))
        blank_ch = 255 * np.ones_like(labels)
        labeled_img = cv2.merge([blank_ch, blank_ch, blank_ch])

        # set bg label to black
        labeled_img[label_hue == 0] = 0

        wholeimage = np.where(labeled_img == 0, wholeimage, labeled_img)


if __name__ == '__main__':
    p = Pool()

    list(p.imap(gabor, enumerate(shp_list)))
    cv2.imwrite(/ccl/streets_gabor_ccl.tiff", wholeimage)

Answer 1

从假设 adjust_gamma 不fork 子进程或使用 C 线程开始，图像处理是 CPU 密集型进程，因此您应该依赖进程而不是 Python 中的线程.

它应该是这样的:

from multiprocessing.pool import Pool

SinglestreetsFolder = "/dop_shapefile/"
shp_list = [x for x in os.listdir(SinglestreetsFolder) if x.endswith("clipped.tiff")]

def process_image(params):
    i, image_name = params
    originalPath = "/dop_shapefile/" + image_name
    original = cv2.imread(originalPath)

    adjusted = adjust_gamma(original, gamma=0.3)

    cv2.imwrite("/gamma/" + image_name + "_gamma.tiff", adjusted)
    return "status_Gamma: ", i + 1, "/", len(shp_list)

if __name__ == '__main__':
    p = Pool()
    results = list(p.imap(process_image, enumerate(shp_list)))
    print results

某些部分可以更改，我只是用尽可能少的更改来调整您的示例。