handDetect

Posted on | In |

使用RealSense进行手势识别

原理

  1. Compute a depth map

    The depth map is constructed by analyzing a speckle pattern of infrared laser light 具体建立结构光的原理是受专利保护的

    • Structured light general principle: project a known pattern onto the scene and infer depth from the deformation of that pattern

    • The Kinect combines structured light with two classic computer vision techniques: depth from focus, and depth from stereo

      • Depth from focus uses the principle that stuff that is more blurry is further away

        越远的物体就会更模糊

        The astigmatic lens causes a projected circle to become an ellipse whose orientation depends on depth

        像散透镜使投影圆变成椭圆，其取向取决于深度

      • Depth from stereo uses parallax

2.infer body postion

代码梳理

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
// C++ Libraries
#include <stdio.h>
#include <iostream>
#include <string>

// OpenCV Libraries
#include <opencv/cxcore.h>
#include "opencv2/highgui/highgui.hpp"

// OpenARK Libraries
#include "Core.h"
#include "RS2Camera.h"
#include "Visualizer.h"

using namespace ark;

int main() {
	// change the SR300Camera to other type if needed
	ark::DepthCamera::Ptr camera = std::make_shared<ark::RS2Camera>();

	ark::DetectionParams::Ptr params = ark::DetectionParams::create();
	//ark::HandDetector handDetector(true, params);
	PlaneDetector::Ptr planeDetector = std::make_shared<PlaneDetector>();
	HandDetector::Ptr handDetector = std::make_shared<HandDetector>(planeDetector);

	//ark::HandDetector handDetector(false, params); // change first argument to true to remove planes in frame
	
	camera->beginCapture();
    //1.step

	int frame = 0;
	while (true)
	{
		// Show image
		cv::Mat xyzVisual; ark::Visualizer::visualizeXYZMap(camera->getXYZMap(), xyzVisual);
		cv::imshow("XYZ Map", xyzVisual);

		// Find hands
		handDetector->update(*camera);
		auto hands = handDetector->getHands();

		if (!hands.empty()) {
			ark::Hand::Ptr hand = hands[0];

			// Show the hand
			cv::Mat visual = camera->getXYZMap().clone();
			ark::Visualizer::visualizeHand(visual, visual, hand.get(), hand->getSVMConfidence());
			cv::imshow("Result", visual);
		}

		/**** Start: Loop Break Condition ****/
		int c = cv::waitKey(1);
		if (c == 'q' || c == 'Q' || c == 27) {
			break;
		}
		/**** End: Loop Break Condition ****/

		++frame;
	}
	return 0;
}

和硬件连接的部分capture

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
    /**
     * Begin capturing frames continuously from this camera on a parallel thread, 
     * capped at a certain maximum FPS.
     * WARNING: throws an error if capture already started.
     * @param fps_cap maximum FPS of capture (-1 to disable)
     * @param removeNoise if true, performs noise removal on the depth image after retrieving it
     * @see endCapture
     * @see isCapturing
     */
    void beginCapture(int fps_cap = -1, bool remove_noise = true);


void DepthCamera::beginCapture(int fps_cap, bool remove_noise)
{
    ASSERT(captureInterrupt == true, "beginCapture: already capturing from this camera");
    captureInterrupt = false;
    std::thread thd(&DepthCamera::captureThreadingHelper, this, fps_cap,
        &captureInterrupt, remove_noise);
    thd.detach();
}
  1. c++多线程
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
// C++ Libraries
#include <stdio.h>
#include <iostream>
#include <string>

// OpenCV Libraries
#include <opencv/cxcore.h>
#include "opencv2/highgui/highgui.hpp"

// OpenARK Libraries
#include "Core.h"
#include "RS2Camera.h"
#include "Visualizer.h"

using namespace ark;

int main() {
	// change the SR300Camera to other type if needed
	ark::DepthCamera::Ptr camera = std::make_shared<ark::RS2Camera>();

	ark::DetectionParams::Ptr params = ark::DetectionParams::create();
	//ark::HandDetector handDetector(true, params);
	PlaneDetector::Ptr planeDetector = std::make_shared<PlaneDetector>();
	HandDetector::Ptr handDetector = std::make_shared<HandDetector>(planeDetector);

	//ark::HandDetector handDetector(false, params); // change first argument to true to remove planes in frame

	camera->beginCapture();

	int frame = 0;
	while (true)
	{
		// Show image
		cv::Mat xyzVisual; ark::Visualizer::visualizeXYZMap(camera->getXYZMap(), xyzVisual);
		cv::imshow("XYZ Map", xyzVisual);

		// Find hands
		handDetector->update(*camera);
		auto hands = handDetector->getHands();

		if (!hands.empty()) {
			ark::Hand::Ptr hand = hands[0];

			// Show the hand
			cv::Mat visual = camera->getXYZMap().clone();
			ark::Visualizer::visualizeHand(visual, visual, hand.get(), hand->getSVMConfidence());
			cv::imshow("Result", visual);
		}

		/**** Start: Loop Break Condition ****/
		int c = cv::waitKey(1);
		if (c == 'q' || c == 'Q' || c == 27) {
			break;
		}
		/**** End: Loop Break Condition ****/

		++frame;
	}
	return 0;
}