face_swap_demo.cpp

#include "opencv2/face.hpp"
#include "opencv2/imgproc.hpp"
#include "opencv2/imgcodecs.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/objdetect.hpp"
#include "opencv2/photo.hpp" // seamlessClone()
#include <iostream>
using namespace cv;
using namespace cv::face;
using namespace std;

static bool myDetector(InputArray image, OutputArray faces, CascadeClassifier *face_cascade)
{
	Mat gray;

	if (image.channels() > 1)
		cvtColor(image, gray, COLOR_BGR2GRAY);
	else
		gray = image.getMat().clone();

	equalizeHist(gray, gray);

	std::vector<Rect> faces_;
	face_cascade->detectMultiScale(gray, faces_, 1.4, 2, CASCADE_SCALE_IMAGE, Size(30, 30));
	Mat(faces_).copyTo(faces);
	return true;
}

void divideIntoTriangles(Rect rect, vector<Point2f> &points, vector< vector<int> > &delaunayTri);
void warpTriangle(Mat &img1, Mat &img2, vector<Point2f> &triangle1, vector<Point2f> &triangle2);

//Divide the face into triangles for warping
void divideIntoTriangles(Rect rect, vector<Point2f> &points, vector< vector<int> > &Tri) {

	// Create an instance of Subdiv2D
	Subdiv2D subdiv(rect);
	// Insert points into subdiv
	for (vector<Point2f>::iterator it = points.begin(); it != points.end(); it++)
		subdiv.insert(*it);
	vector<Vec6f> triangleList;
	subdiv.getTriangleList(triangleList);
	vector<Point2f> pt(3);
	vector<int> ind(3);
	for (size_t i = 0; i < triangleList.size(); i++)
	{
		Vec6f triangle = triangleList[i];
		pt[0] = Point2f(triangle[0], triangle[1]);
		pt[1] = Point2f(triangle[2], triangle[3]);
		pt[2] = Point2f(triangle[4], triangle[5]);
		if (rect.contains(pt[0]) && rect.contains(pt[1]) && rect.contains(pt[2])) {
			for (int j = 0; j < 3; j++)
				for (size_t k = 0; k < points.size(); k++)
					if (abs(pt[j].x - points[k].x) < 1.0 && abs(pt[j].y - points[k].y) < 1)
						ind[j] = (int)k;
			Tri.push_back(ind);
		}
	}
}
void warpTriangle(Mat &img1, Mat &img2, vector<Point2f> &triangle1, vector<Point2f> &triangle2)
{
	Rect rectangle1 = boundingRect(triangle1);
	Rect rectangle2 = boundingRect(triangle2);
	// Offset points by left top corner of the respective rectangles
	vector<Point2f> triangle1Rect, triangle2Rect;
	vector<Point> triangle2RectInt;
	for (int i = 0; i < 3; i++)
	{
		triangle1Rect.push_back(Point2f(triangle1[i].x - rectangle1.x, triangle1[i].y - rectangle1.y));
		triangle2Rect.push_back(Point2f(triangle2[i].x - rectangle2.x, triangle2[i].y - rectangle2.y));
		triangle2RectInt.push_back(Point((int)(triangle2[i].x - rectangle2.x), (int)(triangle2[i].y - rectangle2.y))); // for fillConvexPoly
	}
	// Get mask by filling triangle
	Mat mask = Mat::zeros(rectangle2.height, rectangle2.width, CV_32FC3);
	fillConvexPoly(mask, triangle2RectInt, Scalar(1.0, 1.0, 1.0), 16, 0);
	// Apply warpImage to small rectangular patches
	Mat img1Rect;
	img1(rectangle1).copyTo(img1Rect);
	Mat img2Rect = Mat::zeros(rectangle2.height, rectangle2.width, img1Rect.type());
	Mat warp_mat = getAffineTransform(triangle1Rect, triangle2Rect);
	warpAffine(img1Rect, img2Rect, warp_mat, img2Rect.size(), INTER_LINEAR, BORDER_REFLECT_101);
	multiply(img2Rect, mask, img2Rect);
	multiply(img2(rectangle2), Scalar(1.0, 1.0, 1.0) - mask, img2(rectangle2));
	img2(rectangle2) = img2(rectangle2) + img2Rect;
}

int main(int argc, char** argv)
{
	Mat img1 = imread("D:/vcprojects/images/yuan_test.png");
	Mat img2 = imread("D:/test.png");
	if (img1.empty() || img2.empty()) {
		return -1;
	}
	string modelfile_name = "D:/vcprojects/images/lbfmodel.yaml";
	string cascade_name = "D:/opencv-3.4/opencv/build/etc/lbpcascades/lbpcascade_frontalface.xml";

	// 人脸检测与Landmark68个关键点检测
	CascadeClassifier face_cascade;
	face_cascade.load(cascade_name);
	FacemarkLBF::Params params;
	params.n_landmarks = 68; // 68个标注点
	params.initShape_n = 10;
	params.stages_n = 5; // 算法的5个强化步骤
	params.tree_n = 6; // 模型中每个标注点结构树 数目
	params.tree_depth = 5; // 决策树深度

	// 创建LBF landmark 检测器
	Ptr<FacemarkLBF> facemark = FacemarkLBF::create(params);
	facemark->setFaceDetector((FN_FaceDetector)myDetector, &face_cascade);
	facemark->loadModel(modelfile_name);
	cout << "Loaded model" << endl;

	//vector to store the faces detected in the image
	vector<Rect> faces1, faces2;
	vector< vector<Point2f> > shape1, shape2;

	//Detect faces in the current image
	float ratio1 = (float)img1.cols / (float)img1.rows;
	float ratio2 = (float)img2.cols / (float)img2.rows;
	resize(img1, img1, Size((int)(640 * ratio1), (int)(640 * ratio1)), 0, 0, INTER_LINEAR_EXACT);
	resize(img2, img2, Size((int)(640 * ratio2), (int)(640 * ratio2)), 0, 0, INTER_LINEAR_EXACT);

	Mat img1Warped = img2.clone();
	facemark->getFaces(img1, faces1);
	facemark->getFaces(img2, faces2);

	//Initialise the shape of the faces
	facemark->fit(img1, faces1, shape1);
	facemark->fit(img2, faces2, shape2);

	unsigned long numswaps = (unsigned long)min((unsigned long)shape1.size(), (unsigned long)shape2.size());
	for (unsigned long z = 0; z<numswaps; z++) {
		vector<Point2f> points1 = shape1[z];
		vector<Point2f> points2 = shape2[z];
		img1.convertTo(img1, CV_32F);
		img1Warped.convertTo(img1Warped, CV_32F);
		// Find convex hull
		vector<Point2f> boundary_image1;
		vector<Point2f> boundary_image2;
		vector<int> index;
		convexHull(Mat(points2), index, false, false);
		for (size_t i = 0; i < index.size(); i++)
		{
			boundary_image1.push_back(points1[index[i]]);
			boundary_image2.push_back(points2[index[i]]);
		}
		// Triangulation for points on the convex hull
		vector< vector<int> > triangles;
		Rect rect(0, 0, img1Warped.cols, img1Warped.rows);
		divideIntoTriangles(rect, boundary_image2, triangles);
		// Apply affine transformation to Delaunay triangles
		for (size_t i = 0; i < triangles.size(); i++)
		{
			vector<Point2f> triangle1, triangle2;
			// Get points for img1, img2 corresponding to the triangles
			for (int j = 0; j < 3; j++)
			{
				triangle1.push_back(boundary_image1[triangles[i][j]]);
				triangle2.push_back(boundary_image2[triangles[i][j]]);
			}
			warpTriangle(img1, img1Warped, triangle1, triangle2);
		}

		// 计算与生成模板
		vector<Point> hull;
		for (size_t i = 0; i < boundary_image2.size(); i++)
		{
			Point pt((int)boundary_image2[i].x, (int)boundary_image2[i].y);
			hull.push_back(pt);
		}
		Mat mask = Mat::zeros(img2.rows, img2.cols, img2.depth());
		fillConvexPoly(mask, &hull[0], (int)hull.size(), Scalar(255, 255, 255));

		//	无缝克隆 
		Rect r = boundingRect(boundary_image2);
		Point center = (r.tl() + r.br()) / 2;
		Mat output;
		img1Warped.convertTo(img1Warped, CV_8UC3);
		seamlessClone(img1Warped, img2, mask, center, output, NORMAL_CLONE);
		imshow("Face_Swapped", output);
		imwrite("D:/face_swap_demo.png", output);

		waitKey(0);
		destroyAllWindows();
	}
	return 0;
}