masm.cpp

这是个人脸识别程序

// $masm\masm.cpp 1.5 milbo$ build active shape model from shapes and associated images
// Warning: this is raw research code -- expect it to be quite messy.
// milbo Petaluma jun05
//-----------------------------------------------------------------------------
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// A copy of the GNU General Public License is available at
// http://www.r-project.org/Licenses/
//-----------------------------------------------------------------------------

#include "all.hpp"

typedef struct tShapes				// information on all shapes, initialized from shape file by ReadMasmShapes
	{
	int			nPoints;			// number of points (i.e. landmarks) per shape
	int			nShapes;			// number of shapes
	char		sImageDirs[SLEN];	// image directories at head of shape file, separated by semicolons
	SHAPE		AvShape;			// average shape after aligning shapes
	ShapeVec	OrgShapes;			// array[nShapes] of SHAPE:    original shapes from file, but all with same nbr of points as RefShape
	ShapeVec	Shapes;				// array[nShapes] of SHAPE:    OrgShapes, but aligned
	StringVec	TagStrings;			// array[nShapes] of string:   string tag preceding each shape in shape file
	IntVec		TagInts;			// array[nShapes] of unsigned: hex number at start of each of above tags
	IntVec		nUsedLandmarks;		// array[nShapes] of int: 	   number of landmarks used in each shape
	BoolVec		fExtraLEyeLandsUsed;// array[nShapes] of bool: 	   true if MLEye0..MLEye7 landmark are not 0,0 TODO this is a hack used when synthesizing landmarks
									//							   this is needed because synthesized points are already 0,0 by the time we get to CollectProfs()
	BoolVec		fExtraREyeLandsUsed;// array[nShapes] of bool: 	   ditto but for MREye0..MREye7
	const tLand *pLand;				// landmark information for these shapes
	}
tShapes;

typedef cvec<MatVec> MatVecLev;		// a vector of matrices for each point for each level in the image pyramid
typedef cvec<VecVec> VecVecLev;		// a vector of one-dimensional matrices for each level in the image pyramid
typedef cvec<IntVec> IntVecLev;

typedef cvec<MatVecLev> MatVecLevSub;	// a vector of matrices for each sub-profile for each point for each level in the image pyramid
typedef cvec<VecVecLev> VecVecLevSub;	// a vector of one-dimensional matrices for each sub-profile for each level in the image pyramid

typedef struct tStats	// everything you ever wanted to know about the profiles for all shapes and images at all pyramid levels
	{
	IntVecLev 	 nProfs;	// array(nLevs][nPoints]: nbr of profiles for each point
	MatVecLevSub Covars;    // array[nLevs][nPoints][nSubs] of covar mats: each is nPointsPerProf*nPointsPerProf
	VecVecLevSub AvProfs;	// array[nLevs][nPoints][nSubs] of average profile vecs: each is 1 x nPointsPerProf
	MatVecLevSub Profs;		// array[nLevs][nPoints][nSubs] of profile mats: each is nShapes x nPointsPerProf
	}
tStats;

static bool fgVerbose 	  = false;		// -fv command line flag
static clock_t gStartTime = clock();	// time program started

static const char sgUsage[] =
	"Usage: masm [OPTIONS] INFILE.shape\n"
	"\n"
	"Creates OUTFILE.asm from INFILE.shape\n"
	"\n"
	"Options\n\n"
	"-o OUTFILE     output file [default INFILE.asm]\n"
	"\n"
	"-p PATTERN\n"
	"               Load only shapes in INFILE.shape with tags matching case-independent PATTERN.\n"
    "               PATTERN is an egrep style pattern.\n"
	"               Example: -p \"xyz\" loads filenames containing xyz\n"
	"               Example: -p \" m000| m001\" loads filenames beginning with m000 or m001.\n"
	"               Default: NULL i.e. match all shapes\n"
	"\n"
	"-P Mask1 Mask2\n"
	"               Load only shapes which satisfy (Attr & Mask1) == Mask2.\n"
	"               Attr is the hex number at start of the tag, Mask1 and Mask2 are hex.\n"
	"               This filter is applied after -p PATTERN.\n"
	"               Example: -P 2 2 matches faces with glasses (FA_Glasses=2, see atface.hpp).\n"
	"               Example: -P 2 0 matches faces without glasses.\n"
	"               Default: no filter (Mask1=Mask2=0) i.e. match all shapes\n"
	"\n"
	"-S FILE:FileType:iSet:-I:N\n"
	"               Train on the given sample set (don't use with -P or -p flags)\n"
    "               FILE is the sample file\n"
    "               FileType is the file type (\"m\" or \"B\" or \"*\" for all)\n"
    "               iSet is the column in iSamples (0 is test set, 1 is 1st cross-validation set, etc.)\n"
    "               -I is a negative integer from 1 to N which specifies which part of the validation set to leave out\n"
    "               N for N-fold cross validation\n"
    "               Example: -S m2.sam:*:1:-3:10 uses shapes listed in 1st set of m2.sam\n"
    "                        and leaves out set 3 of 10 (i.e. step 3 of 10-fold cross validation).\n"
	"\n"
	"-fv F          verbose [default 0]\n"
	"\n"
	"-ni N          max number of images [default 0 for all]";

//-----------------------------------------------------------------------------
static char *sSecsElapsed (void)	// return a string showing seconds elapsed since gStartTime
{
static char s[PLEN];
sprintf(s, "[%.1f secs]", (double)(clock() - gStartTime) / CLOCKS_PER_SEC);
return s;
}

//-----------------------------------------------------------------------------
// Add Shape to NewShape on a point by point basis.
// Same as NewShape += Shape, but with this difference: if a point isn't used in Shape,
// we use the corresponding point in the alternative shape AltShape instead

static void SumShapes (SHAPE &NewShape, 							// io
					   const SHAPE &Shape, const SHAPE &AltShape)	// in
{
CheckSameDim(NewShape, Shape, "SumShapes");
CheckSameDim(NewShape, AltShape, "SumShapes");

for (int iPoint = 0; iPoint < Shape.nrows(); iPoint++)
	{
	if (fPointUsed(Shape, iPoint))
		{
		NewShape(iPoint, VX) += Shape(iPoint, VX);
		NewShape(iPoint, VY) += Shape(iPoint, VY);
		}
	else
		{
		NewShape(iPoint, VX) += AltShape(iPoint, VX);
		NewShape(iPoint, VY) += AltShape(iPoint, VY);
		}
	}
}

//-----------------------------------------------------------------------------
// Generate a bitmap with the centralized shapes drawn on it.
// For debugging.

#if CONF_fShowDebugImages
static void ShowCentralizedShapes (const ShapeVec &Shapes, const SHAPE &AvShape)
{
char sPath[SLEN]; sprintf(sPath, "%s/centralized.bmp", CONF_sOutDir);
lprintf(", generating %s", sPath);
int nShapes = Shapes.size();
int iShape;

#if 0
// get image width from shape sizes

int xMax = -1000;
int xMin = 1000;
for (iShape = 0; iShape < nShapes; iShape++)
	{
	int xMin1 = Shapes[iShape].col(VX).minElem();
	if (xMin1 < xMin)
		xMin = xMin1;
	int xMax1 = Shapes[iShape].col(VX).maxElem();
	if (xMax1 < xMax)
		xMax = xMax1;
	}
int nImageWidth = 2 * __max(-xMin + 10, xMax + 10);
nImageWidth = (nImageWidth+4)/4;			// make sure width is divisble by 4
nImageWidth *= 4;
int nImageHeight = 3 * nImageWidth / 2;
#else
int nImageWidth = 500;
int nImageHeight = 500;
#endif

RgbImage Img(nImageWidth, 3 * nImageWidth / 2);
FillRgbImage(Img, 0xff, 0xff, 0xff);
//BresenhamDrawLine(Img, 128, 128, 128, -Img.width/2, 0, Img.width/2, 0);	// draw cross hair
//BresenhamDrawLine(Img, 128, 128, 128, 0, -Img.height/2, 0, Img.height/2);
BresenhamDrawLine(Img, 210, 210, 210, -Img.width/2, 0, Img.width/2, 0);	// draw cross hair
BresenhamDrawLine(Img, 210, 210, 210, 0, -Img.height/2, 0, Img.height/2);

int nShapes1 = __min(20, nShapes);					// print a max of 20 shapes
int nBump = __max(1, (double)nShapes/nShapes1);

#if 0	// just first and last shape

DrawShape(Img, Shapes[0],		   C_RED,    1, IM_NO_CONNECT_DOTS, IM_ANNOTATE, IM_NO_WHISKERS, 0, IM_NO_TRANSPARENT);
DrawShape(Img, Shapes[nShapes-1], C_YELLOW, 1, IM_NO_CONNECT_DOTS, IM_ANNOTATE, IM_NO_WHISKERS, 0, IM_NO_TRANSPARENT);

#else

for (iShape = 0; iShape < nShapes; iShape += nBump)
	{
#if 1
	//char s[100]; sprintf(s, "iShape %d\n", iShape);
	//Shapes[iShape].print(s);
	DrawShape(Img, Shapes[iShape], C_BLACK,    1, IM_CONNECT_DOTS, IM_NO_ANNOTATE, IM_NO_WHISKERS, 0, IM_NO_TRANSPARENT);
#else
	if (iShape == 0 || iShape >= nShapes - nBump) // first and last image in red
		DrawShape(Img, Shapes[iShape], C_RED,    1, IM_CONNECT_DOTS, IM_NO_ANNOTATE, IM_NO_WHISKERS, 0, IM_NO_TRANSPARENT);
	else
		DrawShape(Img, Shapes[iShape], C_YELLOW, 1, IM_CONNECT_DOTS, IM_NO_ANNOTATE, IM_NO_WHISKERS, 0, IM_NO_TRANSPARENT);
#endif
#endif
	}
WriteBmp(Img, sPath);
}
#endif CONF_fShowDebugImages

//-----------------------------------------------------------------------------
static AlignImageToAlignedShape (Image &Img,
										const SHAPE &AlignedShape, const SHAPE &OrgShape)
{
// align image with its aligned shape (use AlignedShapes versus OrgShapes to get needed translation)

int nPoints = AlignedShape.nrows();
SHAPE Shape(OrgShape);
Mat Pose(AlignShape(Shape, AlignedShape));

double xOffset = -Pose(0, 2);
double yOffset = Pose(1, 2);
double Scale = Pose(0, 0);
double Theta = atan(Pose(1, 0) / Pose(0, 0));

// Extend the image so when we extract we don't cut off part of the face
// Needed because ExtractImage doesn't change the size of the image yet extracts a magnified part of the image
// The face in the magnified part of the image could be bigger than the image if we didn't extend.

//TODO could greatly increase the speed of this if we combined the following three func calls into one func

int xExtend = Scale * 2 * abs(xOffset);
int yExtend = Scale * 2 * abs(yOffset);

ExtendImage(Img, xExtend, xExtend, yExtend, yExtend);
ExtractImage(Img, Img.width, Img.height, Img.width/2, Img.height/2, Scale, Theta, QUIET, IM_NEAREST_PIXEL, IM_EXTEND);
MoveImage(Img, -xOffset, yOffset, QUIET, IM_EXTEND);
}

//-----------------------------------------------------------------------------
// Generate a bitmap with all the aligned shapes drawn on it.
// For debugging.

#if CONF_fShowDebugImages
static void ShowAlignedShapes (const tShapes &Shapes, int iRefShape, const char sShapeFile[])	// in
{
#define IMAGE_SCALE 2

Image Img;
RgbImage RgbImg;

lprintf("%s/aligned.bmp ", CONF_sOutDir);

const char *sShapeName = &Shapes.TagStrings[iRefShape][FNAME_OFFSET];
sLoadImageGivenDirs(Img, Shapes.sImageDirs, sShapeName);
AlignImageToAlignedShape(Img, Shapes.Shapes[iRefShape], Shapes.OrgShapes[iRefShape]);
ScaleImage(Img, IMAGE_SCALE * Img.width, IMAGE_SCALE * Img.height, QUIET, IM_NEAREST_PIXEL);
ConvertGrayImageToRgb(RgbImg, Img);

BresenhamDrawLine(RgbImg, 128, 128, 128, -RgbImg.width/2, 0, RgbImg.width/2, 0);	// draw cross hair
BresenhamDrawLine(RgbImg, 128, 128, 128, 0, -RgbImg.height/2, 0, RgbImg.height/2);

for (int iShape = 0; iShape < Shapes.nShapes; iShape++)
	DrawShape(RgbImg, IMAGE_SCALE * Shapes.Shapes[iShape], C_YELLOW, 1.0,
		IM_CONNECT_DOTS, IM_NO_ANNOTATE, IM_NO_WHISKERS, 0, IM_TRANSPARENT);

DrawShape(RgbImg, IMAGE_SCALE * Shapes.AvShape, C_RED, 1.0,
		IM_CONNECT_DOTS, IM_NO_ANNOTATE, IM_NO_WHISKERS, 0, IM_NO_TRANSPARENT, NULL, NULL, IM_DRAW_CIRCLES);

CropImageToFace(RgbImg, Shapes.AvShape, IMAGE_SCALE, IM_WIDE_MARGINS);
RgbPrintf(RgbImg, 8, 3, C_YELLOW, 120, "%s aligned shapes",  sShapeFile);
char sPath[SLEN];
sprintf(sPath, "%s/aligned.bmp", CONF_sOutDir);
WriteBmp(RgbImg, sPath);

#undef IMAGE_SCALE
}
#endif CONF_fShowDebugImages

//-----------------------------------------------------------------------------
// Check that we have enough data for each landmark and issue a warning (but do nothing
// else) if the amount of data is suspiciously low.
// This is needed as a sanity check if we are experimenting with shapes files.
// For example if some landmarks are missing by mistake or we are supposed be generating landmarks
// artificially and don't.

static void CountNbrSamplesForEachLandmark (const ShapeVec &Shapes, const SHAPE &RefShape)
{
int iRow;
int nShapes = Shapes.size();
IntVec nUsed(CONF_nMaxLandmarks, 0);

for (int iShape = 0; iShape < nShapes; iShape++)
	for (iRow = 0; iRow < RefShape.nrows(); iRow++)
		if (Shapes[iShape](iRow, VX) != 0 || Shapes[iShape](iRow, VY) != 0)		// is landmark used?
			nUsed[iRow]++;

for (iRow = 0; iRow < RefShape.nrows(); iRow++)
	{
	if (nUsed[iRow] < 5)														// 5 is arbitrary
		SysErr("Only %d used samples in iRow %d", nUsed[iRow], iRow);
	if (nUsed[iRow] < 100)														// 100 is arbitrary
		{
		Warn("Only %d used samples in iRow %d", nUsed[iRow], iRow);
		break;																	// only issue a max of one warning
		}
	}
}

//-----------------------------------------------------------------------------
// This aligns the shapes using the method in CootesTaylor 2004 section 4.2
// But we differ from that method as follows:
//
//	 1. we don't normalize AvShape len to 1 (so can put aligned shapes on top of
//      images directly for debugging visualization)
//
//	 2. we don't use tangent space (TODO revisit but Stegmann notes that it doesn't make a practical difference)
//
//   3. We ignore missing (i.e. 0,0) landmarks i.e. we can deal with shapes with missing landmarks
//		This allows you to mix, say, XM2VTS and BioId or AR images during training.
//		The passed in RefShape must have all its landmarks, though.

static void AlignShapes (ShapeVec &Shapes, SHAPE &AvShape, 	// io
						 const SHAPE &RefShape) 			// in
{
#define CONF_nMaxAlignPasses 20

int nShapes = Shapes.size();
for (int iShape = 0; iShape < nShapes; iShape++)
#if CONF_fStraightenBeforeAligning
	StraightenAndCentralizeShape(Shapes[iShape]);
#else
	CentralizeShape(Shapes[iShape]);
#endif

CountNbrSamplesForEachLandmark(Shapes, RefShape); // check that we have enough data for each landmark

// generate AvShape, the average shape

AvShape = RefShape;

for (int iPass = 0; iPass < CONF_nMaxAlignPasses; iPass++)
	{
	SHAPE NewAvShape(AvShape.nrows(), AvShape.ncols());
	SHAPE AltShape(AvShape);	// alternate shape: see SumShapes
	for (iShape = 0; iShape < nShapes; iShape++)
		{
		AlignShape(Shapes[iShape], AvShape);		//TODO was CentralizeShape(Shapes[iShape])