facenet.cpp

Gaussian Mixture Algorithm

		intg sj = inx.dim(2);
		int i = 0, j = 0;
		double scale = (double) (s0j / sj);
		{ idx_bloop1(re, *((Idx<double>*) r.get()), double) {
			j = 0;
			{ idx_bloop1(ree, re, double) {
				if (ree.get(1) > threshold) {
					intg ri = rlist.dim(0);
					rlist.resize(ri + 1, rlist.dim(1));
					rlist.set(ree.get(0), ri, 0);
					rlist.set(ree.get(1), ri, 1);
					rlist.set((23 + (4 * j)) * scale, ri, 2);
					rlist.set((23 + (4 * i)) * scale, ri, 3);
					rlist.set(objsize * scale, ri, 4);
					rlist.set(objsize * scale, ri, 5);
				}
				j++;
			}}
			i++;
		}}
	}}
	return rlist;
}

Idx<ubyte> facenet::multi_res_prep(Idx<ubyte> *img, float zoom) {
	//! copy input images locally
	idx_copy(*img, grabbed);
	//! prepare multi resolutions input
	Idx<double> inx;
	int ni = ((state_idx*) inputs.get(0))->x.dim(1);
	int nj = ((state_idx*) inputs.get(0))->x.dim(2);
	int zi = max(ni, (int) (zoom * grabbed.dim(0)));
	int zj = max(nj, (int) (zoom * grabbed.dim(1)));
	int oi = (zi - ni) / 2;
	int oj = (zj - nj) / 2;
	Idx<ubyte> im = image_resize(grabbed, zj, zi, 1);
	im = im.narrow(0, ni, oi);
	im = im.narrow(1, nj, oj);
	//! for display
	idx_clear(grabbed);
	Idx<ubyte> display(grabbed.narrow(0, im.dim(0), 0));
	display = display.narrow(1, im.dim(1), 0);
	idx_copy(im, display);
	{ idx_bloop1(in, inputs, void*) {
		inx = ((state_idx*) in.get())->x;
		ni = inx.dim(1);
		nj = inx.dim(2);
		Idx<ubyte> imres = image_resize(im, nj, ni, 1);
		Idx<double> inx0 = inx.select(0, 0);
		Idx<double> inx1 = inx.select(0, 1);
		idx_copy(imres, inx0);
		idx_copy(imres, inx1);
		//! high-pass filtering
		for(int layer = 0; layer <= 1; layer++){
			Idx<double> big(ni + 4, nj + 4);
			idx_fill(big, 0.0);
			Idx<double> bla = inx.select(0, layer);
			Idx<double> bla2 = big.narrow(0, ni, 2).narrow(1, nj, 2);
			idx_copy( bla, bla2);
			idx_2dconvol(big, highpass_kernel, bla);
		}
		//! normalization
		double bias = idx3_mean(&inx);
		double coeff = idx3_coef(&inx, bias);
		idx_addc(inx, -bias, inx);
		idx_dotc(inx, 1/coeff, inx);
	}}
	return display;
}

Idx<double> facenet::multi_res_fprop(double threshold, int objsize) {
	//! fprop network on different resolutions
	{ idx_bloop2(in, inputs, void*, out, outputs, void*) {
		fprop((state_idx*) in.get(), (state_idx*) out.get());
		}}
	//! post process outputs
	Idx<double> res = postprocess_output(threshold, objsize);
	//! prune results
	res = prune(res);
	//! print results
	cout << " results: \n";
	{ idx_bloop1(re, res, double) {
		re.printElems();
		cout << " " << labels->get((int)re.get(0));
	}}
	if (res.dim(0) == 0) cout << "no object found.";
	cout << "\n";
	return res;
}

Idx<double> facenet::prune(Idx<double> &res, float ratio) {
	//! prune a list of detections.
	//! only keep the largest scoring within an area
	Idx<double> rlist(1, res.dim(1));
	rlist.resize(0, rlist.dim(1));
	{ idx_bloop1(re, res, double) {
		double score = re.get(1);
		int px = re.get(2);
		int py = re.get(3);
		int bx = re.get(4);
		int by = re.get(5);
    //! if the rectangle overlaps an other rectangle with higher score more than ratio, kill it
		bool ok = true;
		{ idx_bloop1(o, res, double) {
			double scoreo = o.get(1);
			int pxo = o.get(2);
			int pyo = o.get(3);
			int bxo = o.get(4);
			int byo = o.get(5);
		  if ((score < scoreo) &&
		  		((common_area(pxo - (0.5 * bxo),	pyo - (0.5 * byo), bxo, byo, px - 0.5 * bx, py - 0.5 * by, bx, by) > ratio)
		  		|| (common_area(px - 0.5 * bx, py - 0.5 * by, bx, by, pxo - (0.5 * bxo),	pyo - (0.5 * byo), bxo, byo) > ratio)))
		  	ok = false;
		}}
		if (ok) {
			intg ri = rlist.dim(0);
			rlist.resize(ri + 1, rlist.dim(1));
			Idx<double> out(rlist.select(0, ri));
			idx_copy(re, out);
		}
	}}
	return rlist;
}

Idx<int> facenet::calc_sizes(int height, int width){
	int h = (height - 38) / 4;
	int w = (width - 38) / 4;
	int s = min(h, w);
	int sizes[50];
	int n = 0;
	while( s != 0){
		sizes[n]= s;
		s = s / sqrt(2);
		n++;
	}
	Idx<int> sz(n);
	for(int i = 0; i < n; i++) sz.set(sizes[i], i);
	return sz;
}

void facenet::show_net(bool kernel, bool prop){
	if(kernel){
		ebbox* kernels = new ebbox(0, "kernels");
		Idx_Gui* c0kernel = new Idx_Gui(&(this->c0_module->kernel->x), DOUBLE, "c0-layer");
		kernels->add_box( c0kernel);
		Idx_Gui* c1kernel = new Idx_Gui(&(this->c1_module->kernel->x), DOUBLE, "c1-layer");
		kernels->add_box( c1kernel);
		Idx_Gui* c2kernel = new Idx_Gui(&(this->c2_module->kernel->x), DOUBLE, "c2-layer");
		kernels->add_box( c2kernel);
		kernels->show();
	}
	if(prop){
		ebbox* eprop = new ebbox(0, "propagation");
		Idx_Gui* c0prop = new Idx_Gui((void*)&(this->c0_state->x), DOUBLE, "c0-prop");
		eprop->add_box( c0prop);
		Idx_Gui* c1prop = new Idx_Gui((void*)&(this->c1_state->x), DOUBLE, "c1-prop");
		eprop->add_box( c1prop);
		Idx_Gui* c2prop = new Idx_Gui((void*)&(this->c2_state->x), DOUBLE, "c2-prop");
		eprop->add_box( c2prop);
		Idx_Gui* oprop = new Idx_Gui((void*)&(((state_idx*)(outputs.get(0)))->x), DOUBLE, "outputs");
		eprop->add_box( oprop);
		eprop->show();
	}
}


//////////////////////////////////////////////////////

void extract_faces(int height, int width, const string imgDir, const string coordFile, int LineToSkip, char delim, string output, const string outputcoordfile){
	string myfilename = imgDir + "/" + coordFile;
	ifstream myfile(myfilename.c_str());
	if(!myfile.is_open()){
		cout << "Could not open the file :" << myfilename <<"\n";
		return;
	}
	Idx<ubyte> out(1, height, width);
	if((output != "")&&(!load_matrix(out, output.c_str()))){
		cout << "Could not open the output matrix :" << output <<"\n";
		return;
	}
	int index = out.dim(0)-1;

	ofstream myoutputfile;
	myoutputfile.open(outputcoordfile.c_str(), ios_base::ate);

	char* newline = new char[256];
	// skip the irrelevent first lines
	for(int i = 0; i < LineToSkip; i++) myfile.getline(newline, 256);

	char* namefile, *nameandpath;
	int x_centereye, y_centereye, x_centermouth, y_centermouth;
	int x_centereye_o, y_centereye_o, x_centermouth_o, y_centermouth_o;

	dseed(time(0));
	int blabla = 0;
	while(!myfile.eof()){
		newline = new char[256];
		myfile.getline(newline, 256);
		namefile = new char[256];
		int* points = lineparser(newline, delim, namefile);
		if(points == NULL) continue;

		Idx<ubyte> image(1,1,1);
		nameandpath = new char[256];
		string bla = imgDir + "/";
		strcpy(nameandpath, bla.c_str());
		strcat(nameandpath, namefile);
		// open the image in an Idx<ubyte>
		if(image_read_rgbx(nameandpath, image)){
			cout << blabla++ << endl;
			x_centereye_o = x_centereye = points[0];
			y_centereye_o = y_centereye = points[1];
			x_centermouth_o = x_centermouth = points[2];
			y_centermouth_o = y_centermouth = points[3];

			// calculate the angle to verticality of the face, and its size.
			// coeff is a subsampling coefficient used if the face is to big
			// facesize is the nb of pixels between the eyes and the mouth in the final training images.
			int facesize = 8;
			int dist = (x_centermouth - x_centereye)*(x_centermouth - x_centereye) + (y_centermouth - y_centereye)*(y_centermouth - y_centereye);
			double temp1 = 3.1415 / 180 *(x_centermouth - x_centereye)/ sqrt((double)dist);
			double angle = asin(temp1);
			double coeff = facesize / sqrt((double)dist);
			int nsub = 0;
			while(coeff < 0.5){
				Idx<ubyte> image2 = image_subsample(image, 2, 2);
				image.resize(image2.dim(0), image2.dim(1), image2.dim(2));
				idx_copy(image2, image);
				x_centereye /= 2;
				y_centereye /= 2;
				x_centermouth /= 2;
				y_centermouth /=2;
				dist = (x_centermouth - x_centereye)*(x_centermouth - x_centereye) + (y_centermouth - y_centereye)*(y_centermouth - y_centereye);
				coeff = facesize / sqrt((double)dist);
				nsub++;
			}

			// for each face, 10 images will be created by scaling and rotating the face randomly,
			// and 10 more by flipping the first 10
			for(int j = 0; j < 10; j++){
				double rand_angle = drand(30 - abs((int)angle));
				double rand_coeff = drand(1/sqrt(sqrt((double)2)), sqrt(sqrt((double)2)));
				Idx<intg> wh(2);
				Idx<ubyte> bg(4);
				Idx<double> cxcy(2);
				// rotate (up to a 30 angle) and rescale (from a sqrt(sqrt(2)) factor to a 1/sqrt(sqrt(2)) factor) the image
				image_rotscale_rect((int)image.dim(1), (int)image.dim(0), (double)x_centereye, (double)y_centereye, rand_angle + angle, rand_coeff * coeff, wh, cxcy);
				Idx<ubyte> img_rotscaled(wh.get(1), wh.get(0), 3);
				image_rotscale(image, img_rotscaled, x_centereye, y_centereye, cxcy.get(0), cxcy.get(1), rand_angle + angle, rand_coeff * coeff, bg);
				int x_topleft = max((int)cxcy.get(0) - width/2 + (int)drand(2), 0);
				int y_topleft = max((int)cxcy.get(1) - height/2 + (int)drand(2), 0);
				// if a width*height image can be cropped from the resulting image, then crop it and save the according data
				if((x_topleft + width < wh.get(0))&&(y_topleft + height < wh.get(1))){
					Idx<ubyte> img_cropped = image_crop(img_rotscaled, x_topleft, y_topleft, width, height);
					if(index == out.dim(0)) out.resize(out.dim(0) + 2, out.dim(1), out.dim(2));
					Idx<ubyte> bla1 = img_cropped.select( 2, 0);
					Idx<ubyte> bla2 = out.select(0, index);
					idx_copy(bla1, bla2);
					if(outputcoordfile != ""){
						myoutputfile << index << " " << nameandpath;
						myoutputfile << " " << x_centereye_o ;
						myoutputfile << " " << y_centereye_o;
						myoutputfile << " " << x_centermouth_o;
						myoutputfile << " " << y_centermouth_o;
						myoutputfile << " " << (rand_angle + angle);
						myoutputfile << " " << (coeff * sqrt(nsub) * rand_coeff);
						myoutputfile << " false\n ";
					}
					index++;
					Idx<ubyte> img_flipped(img_cropped.dim(0), img_cropped.dim(1));
					for(int i = 0; i < img_cropped.dim(0); i++)
						for(int j = 0; j < img_cropped.dim(1); j++){
							img_flipped.set(img_cropped.get(i, (width - 1) - j, 0), i, j);
						}
					if(index == out.dim(0)) out.resize(out.dim(0) + 1, out.dim(1), out.dim(2));
					Idx<ubyte> bla3 = out.select(0, index);
					idx_copy(img_flipped, bla3);
					if(outputcoordfile != ""){
						myoutputfile << index << " " << nameandpath;
						myoutputfile << " " << x_centereye_o ;
						myoutputfile << " " << y_centereye_o;
						myoutputfile << " " << x_centermouth_o;
						myoutputfile << " " << y_centermouth_o;
						myoutputfile << " " << (rand_angle + angle) * (nsub + 1);
						myoutputfile << " " << (coeff * sqrt(nsub) * rand_coeff);
						myoutputfile << " true\n ";
					}
					index++;
				}
			}

		}
		delete [] newline;
		//delete [] namefile;
		delete [] nameandpath;
	}
	myfile.close();
	if(outputcoordfile != "") myoutputfile.close();
	if(output != "") save_matrix(out, output.c_str());
	else save_matrix(out, "data/database_faces.mat");

}

int* lineparser(const char* newline, char delim, char* namefile){
	//! this parser has been written to extract information from lines of the following type :
	//! "name_of_the_imge_file x_lefteye ylefteye x_righteye y_righteye x_nose y_nose x_leftcornermouth y_leftcornermouth x_centermouth y_centermouth x_rightcornermouth y_rightcornermouth"
	int x_centereye, y_centereye, x_centermouth, y_centermouth;

	//! find the name of the image to open
	const char* mychar2 = newline;
	char* mychar = strchr(mychar2,delim);
	if(mychar == NULL) return NULL;
	int longueur = mychar - mychar2;
	strncpy(namefile, mychar2, longueur);
	*(namefile + longueur) = 0;
	mychar2 = ++mychar;

	//! find the various points needed to extract the image
	int x_lefteye, y_lefteye, x_righteye, y_righteye;
	{
		mychar = strchr(mychar2,delim);
		if(mychar == NULL) return NULL;
		longueur = mychar - mychar2;
		if(longueur > 256) return NULL;
		char temp[256];
		strncpy(temp, mychar2, longueur);
		*(temp+longueur) = 0;
		x_lefteye = atoi(temp);
		mychar2 = ++mychar;
	}
	{
		mychar = strchr(mychar2,delim);
		if(mychar == NULL) return NULL;
		longueur = mychar - mychar2;
		if(longueur > 256) return NULL;
		char temp[256];
		strncpy(temp, mychar2, longueur);
		*(temp+longueur) = 0;
		y_lefteye = atoi(temp);
		mychar2 = ++mychar;
	}
	{
		mychar = strchr(mychar2,delim);
		if(mychar == NULL) return NULL;
		longueur = mychar - mychar2;
		if(longueur > 256) return NULL;
		char temp[256];
		strncpy(temp, mychar2, longueur);
		*(temp+longueur) = 0;
		x_righteye = atoi(temp);
		mychar2 = ++mychar;
	}
	{
		mychar = strchr(mychar2,delim);
		if(mychar == NULL) return NULL;
		longueur = mychar - mychar2;
		if(longueur > 256) return NULL;
		char temp[256];
		strncpy(temp, mychar2, longueur);
		*(temp+longueur) = 0;
		y_righteye = atoi(temp);
		mychar2 = ++mychar;
	}

	x_centereye = (x_lefteye + x_righteye)/2;
	y_centereye = (y_lefteye + y_righteye)/2;

	for(int i = 5; i<9; i++){
		mychar = strchr(mychar2,delim);
		if(mychar == NULL) return NULL;
		mychar2 = ++mychar;
	}

	{
		mychar = strchr(mychar2,delim);
		if(mychar == NULL) return NULL;
		longueur = mychar - mychar2;
		if(longueur > 256) return NULL;
		char temp[256];
		strncpy(temp, mychar2, longueur);
		*(temp+longueur) = 0;
		x_centermouth = atoi(temp);
		mychar2 = ++mychar;
	}

	{
		char temp[5];
		if(strlen(mychar2) <= 4) return NULL;
		strncpy(temp, mychar2, 4);
		*(temp+4) = 0;
		y_centermouth = atoi(temp);
	}

	int* bla = new int[4];
	bla[0] = x_centereye;
	bla[1] = y_centereye;
	bla[2] = x_centermouth;
	bla[3] = y_centermouth;
	return bla;
}