python - 使用 pylibfreenect2 中的 getPointXYZ() 时，Kinect V2 深度图像会产生奇怪的点云坐标？

标签 python kinect libfreenect2

我正在尝试从我的 MS Kinect V2 生成真实世界坐标。

我已经成功拼凑出 pyqt + opengl 散点图，并使用 pylibfreenect2 显示来自 Kinect 的深度数据。

我立即注意到深度数据与点云数据不同。请注意，我房间的天花板非常扭曲(本来应该是平坦的天花板开始像曲棍球棒图)

绘制深度框架的结果 enter image description here

经过一些阅读和挖掘源文件后，我设法找到了一个看起来非常有前途的函数。

getPointXYZ - Construct a 3-D point in a point cloud.

由于它一次仅适用于一个像素，因此我编写了一个简单的嵌套 for 循环。在下面的代码中，您应该看到以下几行:

out = np.zeros((d.shape[0]*d.shape[1], 3)) #shape = (217088, 3)
for row in range(d.shape[0]):
    for col in range(d.shape[1]):
        world = registration.getPointXYZ(undistorted, row, col) #convert depth pixel to real-world coordinate
        out[row + col] = world

getPointXYZ() 的坐标结果 enter image description here

不确定那里发生了什么。它看起来更像是一条直线，有时它类似于一个矩形，而且非常平坦(但它在所有三个维度上都位于任意角度)。当我将手移到传感器前面时，我可以看到一些点在移动，但看不到可声明的形状。看起来所有的点都挤在一起了。

以下是一个 Python 脚本，它将显示包含 openGL 散点图的 pyQt 应用程序窗口。通过 pylibfreenect2 从 Kinect 传感器接收帧，通过迭代深度数据的每一行和每一列并通过 getPointXYZ 发送它来生成散点图的点(这非常慢并且不起作用......)。

# coding: utf-8

# An example using startStreams
from pyqtgraph.Qt import QtCore, QtGui
import pyqtgraph.opengl as gl

import numpy as np
import cv2
import sys
from pylibfreenect2 import Freenect2, SyncMultiFrameListener
from pylibfreenect2 import FrameType, Registration, Frame, libfreenect2

fn = Freenect2()
num_devices = fn.enumerateDevices()
if num_devices == 0:
    print("No device connected!")
    sys.exit(1)

serial = fn.getDeviceSerialNumber(0)
device = fn.openDevice(serial)

types = 0
types |= FrameType.Color
types |= (FrameType.Ir | FrameType.Depth)
listener = SyncMultiFrameListener(types)

# Register listeners
device.setColorFrameListener(listener)
device.setIrAndDepthFrameListener(listener)

device.start()

# NOTE: must be called after device.start()
registration = Registration(device.getIrCameraParams(),
                            device.getColorCameraParams())

undistorted = Frame(512, 424, 4)
registered = Frame(512, 424, 4)


#QT app
app = QtGui.QApplication([])
w = gl.GLViewWidget()
w.show()
g = gl.GLGridItem()
w.addItem(g)

#initialize some points data
pos = np.zeros((1,3))

sp2 = gl.GLScatterPlotItem(pos=pos)
w.addItem(sp2)


def update():
    frames = listener.waitForNewFrame()

    ir = frames["ir"]
    color = frames["color"]
    depth = frames["depth"]

    d = depth.asarray()

    registration.apply(color, depth, undistorted, registered)

    #There are 3 optionally commented methods for generating points data (the last one is not commented here). 
    #First will generate points using depth data only. 
    #Second will generate colored points and pointcloud xyz coordinates. 
    #Third is simply the pointcloud xyz coordinates without the color mapping. 

    """
    #Format depth data to be displayed
    m, n = d.shape
    R, C = np.mgrid[:m, :n]
    out = np.column_stack((d.ravel() / 4500, C.ravel()/m, (-R.ravel()/n)+1))
    """

    """
    #Format undistorted and regisered data to real-world coordinates with mapped colors (dont forget color=out_col in setData)
    out = np.zeros((d.shape[0]*d.shape[1], 3)) #shape = (217088, 3)
    out_col = np.zeros((d.shape[0]*d.shape[1], 3)) #shape = (217088, 3)
    for row in range(d.shape[0]):
        for col in range(d.shape[1]):
            world = registration.getPointXYZRGB(undistorted, registered, row, col)
            out[row + col] = world[0:3]
            out_col[row + col] = np.array(world[3:6]) / 255

    """

    # Format undistorted data to real-world coordinates
    out = np.zeros((d.shape[0]*d.shape[1], 3)) #shape = (217088, 3)
    for row in range(d.shape[0]):
        for col in range(d.shape[1]):
            world = registration.getPointXYZ(undistorted, row, col)
            out[row + col] = world


    sp2.setData(pos=out, size=2)

    listener.release(frames)

t = QtCore.QTimer()
t.timeout.connect(update)
t.start(50)


## Start Qt event loop unless running in interactive mode.
if __name__ == '__main__':
    import sys
    if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'):
        QtGui.QApplication.instance().exec_()

device.stop()
device.close()

sys.exit(0)

我不确定下一步应该做什么才能获取实际的点云坐标数据。

有人对我做错了什么有什么建议吗？

我的操作系统是 Ubuntu 16.0.4 和 Python 3.5

谢谢。

最佳答案

答案实际上是为了解决我在这些嵌套循环中犯的错误。我注意到它没有正确索引数组:

#From:
out[row + col]
#To:
out[row * n_columns + col]

顶点现在已准确定位在 3D 空间中，并且一切看起来都很好!

enter image description here

这是修改后且功能齐全的代码:

# coding: utf-8

# An example using startStreams
from pyqtgraph.Qt import QtCore, QtGui
import pyqtgraph.opengl as gl

import numpy as np
import cv2
import sys
from pylibfreenect2 import Freenect2, SyncMultiFrameListener
from pylibfreenect2 import FrameType, Registration, Frame, libfreenect2

fn = Freenect2()
num_devices = fn.enumerateDevices()
if num_devices == 0:
    print("No device connected!")
    sys.exit(1)

serial = fn.getDeviceSerialNumber(0)
device = fn.openDevice(serial)

types = 0
types |= FrameType.Color
types |= (FrameType.Ir | FrameType.Depth)
listener = SyncMultiFrameListener(types)

# Register listeners
device.setColorFrameListener(listener)
device.setIrAndDepthFrameListener(listener)

device.start()

# NOTE: must be called after device.start()
registration = Registration(device.getIrCameraParams(),
                            device.getColorCameraParams())

undistorted = Frame(512, 424, 4)
registered = Frame(512, 424, 4)


#QT app
app = QtGui.QApplication([])
w = gl.GLViewWidget()
w.show()
g = gl.GLGridItem()
w.addItem(g)

#initialize some points data
pos = np.zeros((1,3))

sp2 = gl.GLScatterPlotItem(pos=pos)
w.addItem(sp2)

def update():
    colors = ((1.0, 1.0, 1.0, 1.0))

    frames = listener.waitForNewFrame()

    ir = frames["ir"]
    color = frames["color"]
    depth = frames["depth"]

    d = depth.asarray()

    registration.apply(color, depth, undistorted, registered)

    listener.release(frames)

    """
    #Format raw depth data to be displayed
    m, n = d.shape
    R, C = np.mgrid[:m, :n]
    out = np.column_stack((d.ravel() / 4500, C.ravel()/m, (-R.ravel()/n)+1))
    """


    #Format undistorted and regisered data to real-world coordinates with mapped colors (dont forget color=out_col in setData)
    n_rows = d.shape[0]
    n_columns = d.shape[1]
    out = np.zeros((d.shape[0] * d.shape[1], 3), dtype=np.float64)
    colors = np.zeros((d.shape[0] * d.shape[1], 3), dtype=np.float64)
    for row in range(n_rows):
        for col in range(n_columns):
            X, Y, Z, B, G, R = registration.getPointXYZRGB(undistorted, registered, row, col)
            out[row * n_columns + col] = np.array([X, Y, Z])  # np.array(pt, dtype=np.float64)
            colors[row * n_columns + col] = np.divide([R, G, B], 255)  # np.array(pt, dtype=np.float64)


    """
    #Format undistorted depth data to real-world coordinates
    n_rows = d.shape[0]
    n_columns = d.shape[1]
    out = np.zeros((d.shape[0] * d.shape[1], 3), dtype=np.float64)
    for row in range(n_rows):
        for col in range(n_columns):
            X, Y, Z = registration.getPointXYZ(undistorted, row, col)
            out[row * n_columns + col] = np.array([X, Y, Z])  # np.array(pt, dtype=np.float64)
    """

    sp2.setData(pos=np.array(out, dtype=np.float64), color=colors, size=2)



t = QtCore.QTimer()
t.timeout.connect(update)
t.start(50)


## Start Qt event loop unless running in interactive mode.
if __name__ == '__main__':
    import sys
    if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'):
        QtGui.QApplication.instance().exec_()

device.stop()
device.close()

sys.exit(0)

[编辑]

请参阅This Post欲了解更多信息

关于python - 使用 pylibfreenect2 中的 getPointXYZ() 时，Kinect V2 深度图像会产生奇怪的点云坐标？，我们在Stack Overflow上找到一个类似的问题： https://stackoverflow.com/questions/41225647/

上一篇：Python Webapp2 GAE 邮件发送问题

下一篇：python - 根据多个标签文本查找父标签 - BeautifulSoup

相关文章：

python - 如何避免循环回到代码中的行？

python - 如何在pygame中找到用户输入位置上的按钮并更改其颜色？

java - 点云凸包到网格处理的替代方案

c++ - 在 Kinec 图片 C++ 中检测手和物体的接触

opencv - Libfreenect 错误的深度图

c++ - 使用 libfreenect2 的读/写访问冲突 c++

python - 在python中提取xml标签之间的文本

python - 如何使用 tf.data 数据集在同一批处理上执行多个训练操作

java - 从 3 个 Kinect 相机读取数据时出现问题

©2024 IT工具网  联系我们