opencv - 使用 Tensorflow API 检测到对象时如何在图像中选择 ROI

Question

我正在努力查找图像(静态图像)中张开的手指数量。首先，我使用 tensorflow API 通过 egohands 数据集检测手部，然后使用 output_dict['detection_boxes'] 获取 [ymin, xmin, ymax, xmax] 值。

然后我尝试使用该坐标用 opencv 绘制一个矩形。它起作用了，并在检测到的区域精确地绘制了一个矩形。但当选择与 ROI 相同时，它不起作用。

我卡住的线:

image = Image.open(image_path)
  # the array based representation of the image will be used later in order to prepare the
  # result image with boxes and labels on it.
image_np = load_image_into_numpy_array(image)
  # Expand dimensions since the model expects images to have shape: [1, None, None, 3]
image_np_expanded = np.expand_dims(image_np, axis=0)
  # Actual detection.
output_dict = run_inference_for_single_image(image_np, detection_graph)
  # Visualization of the results of a detection.

h = vis_util.visualize_boxes_and_labels_on_image_array(
  image_np,output_dict['detection_boxes'],output_dict['detection_classes'],output_dict['detection_scores'],
      category_index,instance_masks=output_dict.get('detection_masks'),use_normalized_coordinates=True,line_thickness=8)
  #plt.figure(figsize=IMAGE_SIZE)
im_width, im_height = image.size

ymin = output_dict['detection_boxes'][0][0]*im_height
xmin = output_dict['detection_boxes'][0][1]*im_width
ymax = output_dict['detection_boxes'][0][2]*im_height
xmax = output_dict['detection_boxes'][0][3]*im_width



cv2.rectangle(image_np, (int(xmin),int(ymin)), (int(xmax),int(ymax)), (255,0,0),5)
roi = image_np[int(xmin):int(ymin), int(xmax):int(ymax)]
cv2.rectangle(image_np, (int(xmin),int(ymin)), (int(xmax),int(ymax)), (0,0,255),-1)
#cv2.circle(image_np, (int(xmin),int(xmax)), 55, (0,0,255), -1)
####################################

我不知道自己走的路是否正确。下面是我的完整代码。

# coding: utf-8

# # Object Detection Demo
# Welcome to the object detection inference walkthrough!  This notebook will walk you step by step through the process of using a pre-trained model to detect objects in an image. Make sure to follow the [installation instructions](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/installation.md) before you start.

# # Imports

# In[1]:


import numpy as np
import os
import six.moves.urllib as urllib
import sys
import tarfile
import tensorflow as tf
import zipfile
import math

from collections import defaultdict
from io import StringIO
#from matplotlib import pyplot as plt
from PIL import Image
import cv2

# This is needed since the notebook is stored in the object_detection folder.
sys.path.append("..")
from object_detection.utils import ops as utils_ops

if tf.__version__ < '1.4.0':
  raise ImportError('Please upgrade your tensorflow installation to v1.4.* or later!')


from utils import label_map_util

from utils import visualization_utils as vis_util


# # Model preparation 

# ## Variables
# 
# Any model exported using the `export_inference_graph.py` tool can be loaded here simply by changing `PATH_TO_CKPT` to point to a new .pb file.  
# 
# By default we use an "SSD with Mobilenet" model here. See the [detection model zoo](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md) for a list of other models that can be run out-of-the-box with varying speeds and accuracies.

# In[4]:


# What model to download.
MODEL_NAME = 'hand_inference_graph'
#MODEL_FILE = MODEL_NAME + '.tar.gz'
#DOWNLOAD_BASE = 'http://download.tensorflow.org/models/object_detection/'

# Path to frozen detection graph. This is the actual model that is used for the object detection.
PATH_TO_CKPT = MODEL_NAME + '/frozen_inference_graph.pb'

# List of the strings that is used to add correct label for each box.
PATH_TO_LABELS = os.path.join('hand_inference_graph', 'hand_label_map.pbtxt')

NUM_CLASSES = 90


# ## Download Model

# In[ ]:


'''opener = urllib.request.URLopener()
opener.retrieve(DOWNLOAD_BASE + MODEL_FILE, MODEL_FILE)
tar_file = tarfile.open(MODEL_FILE)
for file in tar_file.getmembers():
  file_name = os.path.basename(file.name)
  if 'frozen_inference_graph.pb' in file_name:
    tar_file.extract(file, os.getcwd()) '''


# ## Load a (frozen) Tensorflow model into memory.

# In[ ]:


detection_graph = tf.Graph()
with detection_graph.as_default():
  od_graph_def = tf.GraphDef()
  with tf.gfile.GFile(PATH_TO_CKPT, 'rb') as fid:
    serialized_graph = fid.read()
    od_graph_def.ParseFromString(serialized_graph)
    tf.import_graph_def(od_graph_def, name='')


# ## Loading label map
# Label maps map indices to category names, so that when our convolution network predicts `5`, we know that this corresponds to `airplane`.  Here we use internal utility functions, but anything that returns a dictionary mapping integers to appropriate string labels would be fine

# In[ ]:


label_map = label_map_util.load_labelmap(PATH_TO_LABELS)
categories = label_map_util.convert_label_map_to_categories(label_map, max_num_classes=NUM_CLASSES, use_display_name=True)
category_index = label_map_util.create_category_index(categories)


# ## Helper code

# In[ ]:


def load_image_into_numpy_array(image):
  (im_width, im_height) = image.size
  return np.array(image.getdata()).reshape(
      (im_height, im_width, 3)).astype(np.uint8)


# # Detection

# In[ ]:


# For the sake of simplicity we will use only 2 images:
# image1.jpg
# image2.jpg
# If you want to test the code with your images, just add path to the images to the TEST_IMAGE_PATHS.
PATH_TO_TEST_IMAGES_DIR = 'pics'
image_path = 'pics/image1.jpg'
# Size, in inches, of the output images.
IMAGE_SIZE = (12, 8)


# In[ ]:


def run_inference_for_single_image(image, graph):
  with graph.as_default():
    with tf.Session() as sess:
      # Get handles to input and output tensors
      ops = tf.get_default_graph().get_operations()
      all_tensor_names = {output.name for op in ops for output in op.outputs}
      tensor_dict = {}
      for key in [
          'num_detections', 'detection_boxes', 'detection_scores',
          'detection_classes', 'detection_masks'
      ]:
        tensor_name = key + ':0'
        if tensor_name in all_tensor_names:
          tensor_dict[key] = tf.get_default_graph().get_tensor_by_name(
              tensor_name)
      if 'detection_masks' in tensor_dict:
        # The following processing is only for single image
        detection_boxes = tf.squeeze(tensor_dict['detection_boxes'], [0])
        detection_masks = tf.squeeze(tensor_dict['detection_masks'], [0])
        # Reframe is required to translate mask from box coordinates to image coordinates and fit the image size.
        real_num_detection = tf.cast(tensor_dict['num_detections'][0], tf.int32)
        detection_boxes = tf.slice(detection_boxes, [0, 0], [real_num_detection, -1])
        detection_masks = tf.slice(detection_masks, [0, 0, 0], [real_num_detection, -1, -1])
        detection_masks_reframed = utils_ops.reframe_box_masks_to_image_masks(
            detection_masks, detection_boxes, image.shape[0], image.shape[1])
        detection_masks_reframed = tf.cast(
            tf.greater(detection_masks_reframed, 0.5), tf.uint8)
        # Follow the convention by adding back the batch dimension
        tensor_dict['detection_masks'] = tf.expand_dims(
            detection_masks_reframed, 0)
      image_tensor = tf.get_default_graph().get_tensor_by_name('image_tensor:0')

      # Run inference
      output_dict = sess.run(tensor_dict,
                             feed_dict={image_tensor: np.expand_dims(image, 0)})

      # all outputs are float32 numpy arrays, so convert types as appropriate
      output_dict['num_detections'] = int(output_dict['num_detections'][0])
      output_dict['detection_classes'] = output_dict[
          'detection_classes'][0].astype(np.uint8)
      output_dict['detection_boxes'] = output_dict['detection_boxes'][0]
      output_dict['detection_scores'] = output_dict['detection_scores'][0]
      if 'detection_masks' in output_dict:
        output_dict['detection_masks'] = output_dict['detection_masks'][0]
  return output_dict


# In[ ]:



image = Image.open(image_path)
  # the array based representation of the image will be used later in order to prepare the
  # result image with boxes and labels on it.
image_np = load_image_into_numpy_array(image)
  # Expand dimensions since the model expects images to have shape: [1, None, None, 3]
image_np_expanded = np.expand_dims(image_np, axis=0)
  # Actual detection.
output_dict = run_inference_for_single_image(image_np, detection_graph)
  # Visualization of the results of a detection.

h = vis_util.visualize_boxes_and_labels_on_image_array(
  image_np,output_dict['detection_boxes'],output_dict['detection_classes'],output_dict['detection_scores'],
      category_index,instance_masks=output_dict.get('detection_masks'),use_normalized_coordinates=True,line_thickness=8)
  #plt.figure(figsize=IMAGE_SIZE)
im_width, im_height = image.size

ymin = output_dict['detection_boxes'][0][0]*im_height
xmin = output_dict['detection_boxes'][0][1]*im_width
ymax = output_dict['detection_boxes'][0][2]*im_height
xmax = output_dict['detection_boxes'][0][3]*im_width



cv2.rectangle(image_np, (int(xmin),int(ymin)), (int(xmax),int(ymax)), (255,0,0),5)
roi = image_np[int(xmin):int(ymin), int(xmax):int(ymax)]
cv2.rectangle(image_np, (int(xmin),int(ymin)), (int(xmax),int(ymax)), (0,0,255),-1)
#cv2.circle(image_np, (int(xmin),int(xmax)), 55, (0,0,255), -1)
####################################


cv2.imshow('original',cv2.resize(image_np, (800,600)))
cv2.waitKey(0)
cv2.destroyAllWindows()

另请检查我选择投资返回率的方式。

提前致谢。
(我对机器学习和 OpenCV 非常陌生)

Answer 1

roi = image_np[ymin:ymax,xmin:xmax] 尝试这个。 另外，在您的代码中我看到您已将 num_classes 设置为 90，为什么会这样？当你只有一个类(class)时(手，如果我没记错的话)。