如何使用opencvc++拍摄动态视频图像和深度图像

您可以使用kinect for windows SDK来获取帧，然后将它们转换为opencv格式。看看这个代码示例，它在visual studio中做到了这一点(在微软论坛上的这个帖子中可以找到)，不幸的是，我现在没有一个kinect来测试代码：

#include "stdafx.h"
#define COLOR_WIDTH 640    
#define COLOR_HIGHT 480    
#define DEPTH_WIDTH 320    
#define DEPTH_HIGHT 240    
#define SKELETON_WIDTH 640    
#define SKELETON_HIGHT 480    
#define CHANNEL 3
BYTE buf[DEPTH_WIDTH * DEPTH_HIGHT * CHANNEL];
int drawColor(HANDLE h, IplImage* color)    
{
const NUI_IMAGE_FRAME * pImageFrame = NULL;
HRESULT hr = NuiImageStreamGetNextFrame(h, 0, &pImageFrame);
if (FAILED(hr)) 
{
cout << "Get Image Frame Failed" << endl;
return -1;
}
NuiImageBuffer * pTexture = pImageFrame->pFrameTexture;
KINECT_LOCKED_RECT LockedRect;
pTexture->LockRect(0, &LockedRect, NULL, 0);
if (LockedRect.Pitch != 0)
{
BYTE * pBuffer = (BYTE*) LockedRect.pBits;
cvSetData(color, pBuffer, LockedRect.Pitch);
}
cvShowImage("color image", color);
NuiImageStreamReleaseFrame(h, pImageFrame);
return 0;
}
int drawDepth(HANDLE h, IplImage* depth)
{
const NUI_IMAGE_FRAME * pImageFrame = NULL;
HRESULT hr = NuiImageStreamGetNextFrame(h, 0, &pImageFrame);
if (FAILED(hr))
{
cout << "Get Image Frame Failed" << endl;
return -1;
}
//  temp1 = depth;
NuiImageBuffer * pTexture = pImageFrame->pFrameTexture;
KINECT_LOCKED_RECT LockedRect;
pTexture->LockRect(0, &LockedRect, NULL, 0);
if (LockedRect.Pitch != 0)
{
USHORT * pBuff = (USHORT*) LockedRect.pBits;
for (int i = 0; i < DEPTH_WIDTH * DEPTH_HIGHT; i++)
{
BYTE index = pBuff[i] & 0x07;
USHORT realDepth = (pBuff[i] & 0xFFF8) >> 3;
BYTE scale = 255 - (BYTE)(256 * realDepth / 0x0fff);
buf[CHANNEL * i] = buf[CHANNEL * i + 1] = buf[CHANNEL * i + 2] = 0;
switch (index)
{
case 0:
buf[CHANNEL * i] = scale / 2;
buf[CHANNEL * i + 1] = scale / 2;
buf[CHANNEL * i + 2] = scale / 2;
break;
case 1:
buf[CHANNEL * i] = scale;
break;
case 2:
buf[CHANNEL * i + 1] = scale;
break;
case 3:
buf[CHANNEL * i + 2] = scale;
break;
case 4:
buf[CHANNEL * i] = scale;
buf[CHANNEL * i + 1] = scale;
break;
case 5:
buf[CHANNEL * i] = scale;
buf[CHANNEL * i + 2] = scale;
break;
case 6:
buf[CHANNEL * i + 1] = scale;
buf[CHANNEL * i + 2] = scale;
break;
case 7:
buf[CHANNEL * i] = 255 - scale / 2;
buf[CHANNEL * i + 1] = 255 - scale / 2;
buf[CHANNEL * i + 2] = 255 - scale / 2;
break;
}
}
cvSetData(depth, buf, DEPTH_WIDTH * CHANNEL);
}
NuiImageStreamReleaseFrame(h, pImageFrame);
cvShowImage("depth image", depth);
return 0;
}
int drawSkeleton(IplImage* skeleton)
{
NUI_SKELETON_FRAME SkeletonFrame;
CvPoint pt[20];
HRESULT hr = NuiSkeletonGetNextFrame(0, &SkeletonFrame);
bool bFoundSkeleton = false;
for (int i = 0; i < NUI_SKELETON_COUNT; i++)
{
if (SkeletonFrame.SkeletonData[i].eTrackingState
== NUI_SKELETON_TRACKED)
{
bFoundSkeleton = true;
}
}
// Has skeletons!
//
if (bFoundSkeleton)
{
NuiTransformSmooth(&SkeletonFrame, NULL);
memset(skeleton->imageData, 0, skeleton->imageSize);
for (int i = 0; i < NUI_SKELETON_COUNT; i++)
{
if (SkeletonFrame.SkeletonData[i].eTrackingState
== NUI_SKELETON_TRACKED)
{
for (int j = 0; j < NUI_SKELETON_POSITION_COUNT; j++)
{
float fx, fy;
NuiTransformSkeletonToDepthImageF(
SkeletonFrame.SkeletonData[i].SkeletonPositions[j],
&fx, &fy);
pt[j].x = (int) (fx * SKELETON_WIDTH + 0.5f);
pt[j].y = (int) (fy * SKELETON_HIGHT + 0.5f);
cvCircle(skeleton, pt[j], 5, CV_RGB(255, 0, 0), -1);
}
cvLine(skeleton, pt[NUI_SKELETON_POSITION_HEAD],
pt[NUI_SKELETON_POSITION_SHOULDER_CENTER],
CV_RGB(0, 255, 0));
cvLine(skeleton, pt[NUI_SKELETON_POSITION_SHOULDER_CENTER],
pt[NUI_SKELETON_POSITION_SPINE], CV_RGB(0, 255, 0));
cvLine(skeleton, pt[NUI_SKELETON_POSITION_SPINE],
pt[NUI_SKELETON_POSITION_HIP_CENTER],
CV_RGB(0, 255, 0));
cvLine(skeleton, pt[NUI_SKELETON_POSITION_HAND_RIGHT],
pt[NUI_SKELETON_POSITION_WRIST_RIGHT],
CV_RGB(0, 255, 0));
cvLine(skeleton, pt[NUI_SKELETON_POSITION_WRIST_RIGHT],
pt[NUI_SKELETON_POSITION_ELBOW_RIGHT],
CV_RGB(0, 255, 0));
cvLine(skeleton, pt[NUI_SKELETON_POSITION_ELBOW_RIGHT],
pt[NUI_SKELETON_POSITION_SHOULDER_RIGHT],
CV_RGB(0, 255, 0));
cvLine(skeleton, pt[NUI_SKELETON_POSITION_SHOULDER_RIGHT],
pt[NUI_SKELETON_POSITION_SHOULDER_CENTER],
CV_RGB(0, 255, 0));
cvLine(skeleton, pt[NUI_SKELETON_POSITION_SHOULDER_CENTER],
pt[NUI_SKELETON_POSITION_SHOULDER_LEFT],
CV_RGB(0, 255, 0));
cvLine(skeleton, pt[NUI_SKELETON_POSITION_SHOULDER_LEFT],
pt[NUI_SKELETON_POSITION_ELBOW_LEFT],
CV_RGB(0, 255, 0));
cvLine(skeleton, pt[NUI_SKELETON_POSITION_ELBOW_LEFT],
pt[NUI_SKELETON_POSITION_WRIST_LEFT],
CV_RGB(0, 255, 0));
cvLine(skeleton, pt[NUI_SKELETON_POSITION_WRIST_LEFT],
pt[NUI_SKELETON_POSITION_HAND_LEFT], CV_RGB(0, 255, 0));
cvLine(skeleton, pt[NUI_SKELETON_POSITION_HIP_CENTER],
pt[NUI_SKELETON_POSITION_HIP_RIGHT], CV_RGB(0, 255, 0));
cvLine(skeleton, pt[NUI_SKELETON_POSITION_HIP_RIGHT],
pt[NUI_SKELETON_POSITION_KNEE_RIGHT],
CV_RGB(0, 255, 0));
cvLine(skeleton, pt[NUI_SKELETON_POSITION_KNEE_RIGHT],
pt[NUI_SKELETON_POSITION_ANKLE_RIGHT],
CV_RGB(0, 255, 0));
cvLine(skeleton, pt[NUI_SKELETON_POSITION_ANKLE_RIGHT],
pt[NUI_SKELETON_POSITION_FOOT_RIGHT],
CV_RGB(0, 255, 0));
cvLine(skeleton, pt[NUI_SKELETON_POSITION_HIP_CENTER],
pt[NUI_SKELETON_POSITION_HIP_LEFT], CV_RGB(0, 255, 0));
cvLine(skeleton, pt[NUI_SKELETON_POSITION_HIP_LEFT],
pt[NUI_SKELETON_POSITION_KNEE_LEFT], CV_RGB(0, 255, 0));
cvLine(skeleton, pt[NUI_SKELETON_POSITION_KNEE_LEFT],
pt[NUI_SKELETON_POSITION_ANKLE_LEFT],
CV_RGB(0, 255, 0));
cvLine(skeleton, pt[NUI_SKELETON_POSITION_ANKLE_LEFT],
pt[NUI_SKELETON_POSITION_FOOT_LEFT], CV_RGB(0, 255, 0));
}
}
}
cvShowImage("skeleton image", skeleton);
return 0;
}
int main(int argc, char * argv[])
{
IplImage* color = cvCreateImageHeader(cvSize(COLOR_WIDTH, COLOR_HIGHT), IPL_DEPTH_8U, 4);
IplImage* depth = cvCreateImageHeader(cvSize(DEPTH_WIDTH, DEPTH_HIGHT),IPL_DEPTH_8U, CHANNEL);
IplImage* skeleton = cvCreateImage(cvSize(SKELETON_WIDTH, SKELETON_HIGHT),IPL_DEPTH_8U, CHANNEL);
cvNamedWindow("color image", CV_WINDOW_AUTOSIZE);
cvNamedWindow("depth image", CV_WINDOW_AUTOSIZE);
cvNamedWindow("skeleton image", CV_WINDOW_AUTOSIZE);
HRESULT hr = NuiInitialize(
NUI_INITIALIZE_FLAG_USES_DEPTH_AND_PLAYER_INDEX
| NUI_INITIALIZE_FLAG_USES_COLOR
| NUI_INITIALIZE_FLAG_USES_SKELETON);
if (hr != S_OK)
{
cout << "NuiInitialize failed" << endl;
return hr;
}
HANDLE h1 = CreateEvent(NULL, TRUE, FALSE, NULL);
HANDLE h2 = NULL;
hr = NuiImageStreamOpen(NUI_IMAGE_TYPE_COLOR, NUI_IMAGE_RESOLUTION_640x480,
0, 2, h1, &h2);
if (FAILED(hr))
{
cout << "Could not open image stream video" << endl;
return hr;
}
HANDLE h3 = CreateEvent(NULL, TRUE, FALSE, NULL);
HANDLE h4 = NULL;
hr = NuiImageStreamOpen(NUI_IMAGE_TYPE_DEPTH_AND_PLAYER_INDEX,
NUI_IMAGE_RESOLUTION_320x240, 0, 2, h3, &h4);
if (FAILED(hr))
{
cout << "Could not open depth stream video" << endl;
return hr;
}
HANDLE h5 = CreateEvent(NULL, TRUE, FALSE, NULL);
hr = NuiSkeletonTrackingEnable(h5, 0);
if (FAILED(hr))
{
cout << "Could not open skeleton stream video" << endl;
return hr;
}
while (1)
{
WaitForSingleObject(h1, INFINITE);
drawColor(h2, color);
WaitForSingleObject(h3, INFINITE);
drawDepth(h4, depth);
WaitForSingleObject(h5, INFINITE);
drawSkeleton(skeleton);
//exit
int c = cvWaitKey(1);
if (c == 27 || c == 'q' || c == 'Q')
break;
}
cvReleaseImageHeader(&depth);
cvReleaseImageHeader(&color);
cvReleaseImage(&skeleton);
cvDestroyWindow("depth image");
cvDestroyWindow("color image");
cvDestroyWindow("skeleton image");
NuiShutdown();
return 0;
}

OpenCV不提供连接Kinect传感器并处理来自Kinect传感器的数据的功能；除非您将Kinect视为常规网络摄像头。您将希望使用其中一个API获取数据并将其发送到OpenCV。要从Kinect获取数据，您可以使用：

• Microsoft Kinect for Windows SDK
• OpenKinect的libfreenect API
• OpenNI+OpenKinect

如果你的雇主对其中一个API有问题，那就有选择了。但是OpenCV的使用并不能消除您使用其中一个的需要。

在MSDN上快速搜索会发现关于这个主题的多个线程。我读过的最直接的方法是在转换图像后使用cvSetData导入数据：

RGB

IplImage * ovImage = NULL;
ovImage = cvCreateImage(cvSize(640, 480), 8, 4);
cvSetData(ovImage, pBuffer, ovImage->widthStep);

深度

ovImage = cvCreateImage(cvSize(640, 480), 8, 1);

我还在GitHub上找到了freenomad_vision项目，该项目提供了对OpenCV和OpenGL的libfreenect支持。如果您不喜欢使用libfreenect，那么代码可以很容易地作为参考，因为传入的数据都是相同的，并且(很可能)会被转换为相同的数据。

如果有人被重定向到这里，寻找一种更简单的方法来可视化Kinect深度流，我可以通过以下方式做到这一点用于KinectV2。

Mat CDepthMap::getFrame()
{
IDepthFrame* frame;
Mat depthImage;
hr = _depth_reader->AcquireLatestFrame(&frame);
if (SUCCEEDED(hr)) {
const UINT imgSize = sDepthWidth*sDepthHeight; //512*424
UINT16 pixelData[imgSize];
hr = frame->CopyFrameDataToArray(imgSize, pixelData);
if (SUCCEEDED(hr)) {
depthImage = Mat(sDepthHeight,sDepthWidth, CV_8U);
for (UINT i = 0; i < imgSize; i++) {
UINT16 depth = pixelData[i];
depthImage.at<UINT8>(i) = LOWORD(depth);
}
}
SafeRelease(frame);
}
return depthImage;
}