gestures.c~

基于Hu矩和支持向量机手完整性检测算法及应用.手势识别

/*
 *   -
 *   Visual extraction of posture information
 *   Copyright (C) <2004,2008>  
 *                       Hector Hugo Aviles-Arriaga
 *    
 *
 *   This program is free software: you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation, either version 3 of the License, or
 *   (at your option) any later version.
 *
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *
 *   You should have received a copy of the GNU General Public License
 *   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 * 
 *   
 */

/**
 *  ----------------------------------------------------------------------------
 *
 *  gestures.c. This file includes all neccesary functions to execute 
 *  the visual extraction of user's postion every 4 images useful in gesture recognition.
 *   
 *  To run the program, please type: ./gestures -f filename.ext
 *
 *  The output must be the file: filename.ext 
 * 
 *  ----------------------------------------------------------------------------
 */

#include "cv.h"
#include "highgui.h"

#include <getopt.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <limits.h>
#include <time.h>
#include <ctype.h>
#include <float.h>

#include "gestures.h"

/**
 * 
 *  Main function 
 *
 */
int main( int argc, char** argv )
{
    // OpenCV stuff 
    CvCapture* capture = NULL; 
    static CvHaarClassifierCascade* face_classifier = NULL;

    // get the command line options 
    get_options( argc, argv, face_classifier_filename);

    // allocate memory for color histograms  
    skin_hist = alloc_histogram_mem();
    non_skin_hist = alloc_histogram_mem();
    face_hist = alloc_histogram_mem();
    torso_hist = alloc_histogram_mem();
    combined_skin_hist = alloc_histogram_mem();
               
    // read general skin and non-skin color models from file
    load_color_model("srgb32_merge_mod.dat", &num_skin_pixels, SKIN);
    load_color_model("nsrgb32_bootstrap.dat", &num_non_skin_pixels, NON_SKIN);

    // compute general skin and non-skin prior probabilities P(Skin) and P(non-skin), respectively, 
    // useful if you want to test the Bayes rule to classify skin pixels 
    /*
    skin_prob = num_skin_pixels / (double)(num_skin_pixels + num_non_skin_pixels);	
    nskin_prob = num_non_skin_pixels / (double)(num_skin_pixels + num_non_skin_pixels);
    */

    // Load Haar face classifier 
    face_classifier = (CvHaarClassifierCascade*)cvLoad(face_classifier_filename, 0, 0, 0 );
    if( !face_classifier)
    {
        fprintf( stderr, "Error: could not load Haar classifier: %s \n", face_classifier_filename);
        exit (1);
    }

    capture = cvCaptureFromCAM( 0 ); // Camera default : 0
    cvNamedWindow( "gestures", CV_WINDOW_AUTOSIZE );
    cvNamedWindow( "backprojection", CV_WINDOW_AUTOSIZE );

    if( capture )
    {
	// image capture loop
        for(;;)
        {
            // Get an image
            frame = cvQueryFrame( capture );

            // try to find the user's face.
	    if (!face){	
               face_detection(face_classifier);

            } else {
                // find the user's right hand
                if ( face && !hand){ 
                   // hand segmentation over the region defined by expected_hand_pos variable 
                   hand_detection();

                } else {	  		
                   // If the right-hand has been found, start tracking 
                   if (hand){
	
                      hand_tracking();

                      // If the hand is outside its initial position
                      // queue the posture data of the user
                      queue_observations();  

                    } // End hand

                } // End face && !hand

            } // End !face

            // draw face and hand tracking results  
            draw();
               
	    cvShowImage( "gestures", frame );

            if( cvWaitKey( 2 ) >= 27 ){
               break;
            }

        }

        cvReleaseCapture( &capture );
    }

    cvReleaseImage( &frame );    

    // free histogram memory
    cvReleaseHist(&skin_hist);
    cvReleaseHist(&non_skin_hist);
    cvReleaseHist(&face_hist);
    cvReleaseHist(&combined_skin_hist);

    cvDestroyWindow("gestures");
    cvDestroyWindow("backprojection");

    return 0;
}

/**
 * 
 *  Allocate memory for a histogram with 32 class intervals for each RGB color channel 
 *   
 */
CvHistogram* alloc_histogram_mem(void)
{
    
    int b_bins = NUM_BINS, g_bins = NUM_BINS, r_bins = NUM_BINS;
    int hist_size[] = {b_bins, g_bins, r_bins};
    float b_ranges[] = { 0, 255 }; 
    float g_ranges[] = { 0, 255 }; 
    float r_ranges[] = { 0, 255 }; 
    float* ranges[] = { b_ranges, g_ranges, r_ranges };

    CvHistogram* hist = cvCreateHist( 3, hist_size, CV_HIST_ARRAY, ranges, 1 );
    cvClearHist( hist );

  return hist;

}

/**
 * 
 *  load  from file a RGB color model with 32 class-intervals
 *  
 */
void load_color_model(char file[30], long int *frequency, int type){

  CvHistogram *hist; 
  FILE *stream; 
  int red, green, blue;
  int val;
  float *ptr; 

    // select the color model to load
    if (type == SKIN){
         hist = skin_hist;
    } else 
    if (type == NON_SKIN){
	 hist = non_skin_hist;
    }

    if ((stream = fopen(file, "rt")) == NULL){ 
          fprintf(stdout, "Error: cannot read the file %s\n", file);
          exit(1);
    }

    fscanf(stream, "%ld", frequency);
        for (red = 0; red < NUM_BINS; red++)
              for (green = 0; green < NUM_BINS; green++)
                     for (blue = 0; blue < NUM_BINS; blue++){
                         fscanf(stream, "%d", &val);
	                 // BGR
		         ptr = cvGetHistValue_3D(hist, blue, green, red);
		         *ptr = val / (float)*frequency;
                     }

      fclose(stream);

      //cvNormalizeHist( hist, 1.0 );
 
}

/**
 * 
 *  Search for the user's face over the upper half of the image 
 *   
 */
void face_detection(CvHaarClassifierCascade *face_classifier)
{
    //static CvMemStorage* face_storage = cvCreateMemStorage(0);

    // set ROI
    cvSetImageROI( frame, cvRect( 0, 0, frame->width, frame->height / 2 ));
    cvClearMemStorage( face_storage );

    CvSeq *faces_seq = cvHaarDetectObjects( frame, face_classifier, 
				face_storage, 1.1, 5, CV_HAAR_DO_CANNY_PRUNING, cvSize(40, 40) );

    cvResetImageROI( frame );

    // it is assumed that it is detected only one face 
    face = (CvRect*)cvGetSeqElem( faces_seq, 0 ); 
    
    if (face){
        // extract skin color from the face
        get_face_skin_color();

        // estimate torso and right-hand subregions using anthropometrical measures first 	
        body_dimensions();

        // get the color histogram for the torso	  	
        get_torso_color();

    }

}

/**
 * 
 *  Fill the face-skin histogram and adjust the rectangle of the face for better fitting  
 *  
 */
void get_face_skin_color(void){

    CvRect subface;

      // create the histogram color model for the face  
      // with pixels taken from a small subregion 
      // TODO: test if it is better to consider the whole face region
      int xsub = (int)(face->width * 0.25);
      int ysub = (int)(face->height * 0.10);

      subface.x = face->x + xsub; 
      subface.y = face->y + ysub; 
      subface.width = face->width - (2 * xsub);
      subface.height = face->height - (2 * ysub);

      // get the color histogram of the face 
      cvSetImageROI( frame, cvRect( subface.x, subface.y, subface.width, subface.height));

      IplImage* b_plane = cvCreateImage( cvGetSize(frame), 8, 1 );
      IplImage* g_plane = cvCreateImage( cvGetSize(frame), 8, 1 );
      IplImage* r_plane = cvCreateImage( cvGetSize(frame), 8, 1 );
      IplImage* planes[] = { b_plane, g_plane, r_plane };

      // BGR pixel order 
      cvCvtPixToPlane( frame, b_plane, g_plane, r_plane, 0 );
      cvCalcHist( planes, face_hist, 0, 0 );
      cvNormalizeHist( face_hist, 1.0 );

      cvReleaseImage( &b_plane ); 
      cvReleaseImage( &g_plane ); 
      cvReleaseImage( &r_plane ); 
 
      cvResetImageROI( frame );

      // adjust the size of the face rectangle for better fitting
      fit();


}


/**
 * 
 *   Adjust the face rectangle
 *  
 */
void fit(void){

   int x, y;
   int *hist;
   int width, height;
   unsigned char color[3];

   cvSetImageROI( frame, *face );

   width = frame->roi->width;
   height = frame->roi->height;

   hist = (int *) malloc(sizeof(int) * width);
   memset(hist, 0, sizeof(int) * width);

   // Sum the number of skin pixels in the columns on the face rectangle 
   for (y = 0; y < height; y++ ){
      for (x = 0; x < width; x++ ){
          get_pixel_color(x, y, color);
          if (cvQueryHistValue_3D(face_hist, color[0]/8, color[1]/8, color[2]/8)){
              hist[x]++;
          }    
      }
   }      

  // adjust the face width according to the number of skin pixels of 
  // each column 
  int xleft = 0;
  do {
    if (hist[xleft] > height / 2) break;
  } while (++xleft < width );

  int xright = width - 1;
  do {
    if (hist[xright] > height / 2) break;
  } while (--xright > xleft );

  free(hist);

  face->x = face->x + xleft;
  face->width = xright - xleft;

  cvResetImageROI( frame );

}

/**
 *   
 *  search for the right-hand over a previously estimated region     
 *  and combine general and particular skin histograms
 *   
 */
void hand_detection(void){
 
   int x, y;
   unsigned char color[3]; // pointer to a image's pixel (in BGR order) 
   float *ptr;   // pointer to update frequency of the combined skin histogram 
   int index_0, index_1, index_2;  
   float general_skin_prob,        // "general" skin probability of a pixel obtained off-line
         general_non_skin_prob,    // "general" non-skin probability of a pixel sampled off-line
         particular_skin_prob,     // "particular" skin probability of a pixel taken from the user's face
         particular_non_skin_prob; // "particular" non-skin probability of a pixel taken from the user's torso

   // Camshift stuff
   CvBox2D track_box;
   CvConnectedComp track_comp;

   // Patch to avoid system crash when the expected position of the hand is
   // estimated beyond the image limits
   if (!((expected_hand_pos.y + expected_hand_pos.height) < frame->height)){  
 
       // Repeat face detection again
       face = NULL;  

   } else {
       // Estimated region of the hand 
       cvSetImageROI( frame, cvRect(expected_hand_pos.x, expected_hand_pos.y, 
					expected_hand_pos.width, expected_hand_pos.height));

       // skin classification over the expected position of the hand
       for (x = 0; x < expected_hand_pos.width; x++){
          for (y = 0; y < expected_hand_pos.height; y++){
               // get the color of the pixel )x,y)    
               get_pixel_color(x, y, color);
               // assume BGR order
               // convert the original color range from [0,255] to [0,32].
               index_0 = (int)(color[0] / 8); 
               index_1 = (int)(color[1] / 8);
               index_2 = (int)(color[2] / 8);

               // query the probabilities for the (x,y) pixel,
               // particular and general skin probabilities first	
               general_skin_prob =  cvQueryHistValue_3D(skin_hist, index_0, index_1, index_2);
               particular_skin_prob = cvQueryHistValue_3D(face_hist, index_0, index_1, index_2);
               // particular and general non-skin probabilities    
               general_non_skin_prob = cvQueryHistValue_3D(non_skin_hist, index_0, index_1, index_2);
               particular_non_skin_prob = cvQueryHistValue_3D(torso_hist, index_0, index_1, index_2);
               // skin classification step		
              // if ((general_skin_prob) > (general_non_skin_prob)){
              if ((particular_skin_prob * general_skin_prob) > (particular_non_skin_prob * general_non_skin_prob)){
                   // NOTE: the combined histogram for skin is constructed  
                   // only with pixels classified as skin over the user's hand
                   // TODO: test if it is better to combine the whole histograms  
                   ptr = cvGetHistValue_3D(combined_skin_hist, index_0, index_1, index_2);
                   *ptr += 1;
                   //*ptr = 1;
               } // end-if

          } // end-for 

       } // end-for

      // Try to segment the hand for the first time
      back_projection ();
      cvCamShift( backproject, cvRect(0, 0, expected_hand_pos.width, expected_hand_pos.height),
                         cvTermCriteria( CV_TERMCRIT_EPS | CV_TERMCRIT_ITER, 10, 1 ), &track_comp, &track_box );
      cvReleaseImage( &backproject );

      if ((track_comp.rect.width > 2) && (track_comp.rect.height > 2)) { 
         // adjust the current hand coordinates to the original image coordinates
         hand = (CvRect *) malloc(sizeof(CvRect));
         hand->x = frame->roi->xOffset + track_comp.rect.x;   
         hand->y = frame->roi->yOffset + track_comp.rect.y;
         hand->width = track_comp.rect.width;
         hand->height = track_comp.rect.height;
 
         // mark the initial position of the right-hand 
         initial_hand_pos.x = (hand->x + (hand->width / 2))  - 30;
         initial_hand_pos.y = (hand->y + (hand->height / 2)) - 30;
         initial_hand_pos.width = 60;    
         initial_hand_pos.height = 60;
 
         cvResetImageROI( frame );
         cvReleaseImage( &backproject );

      }

    } // if