server mod for socket communication with the client

Author: DLX4

face_recogition/detectObject.cpp

@@ -96,7 +96,7 @@ void detectLargestObject(const Mat &img, CascadeClassifier &cascade, Rect &large
     // How much the detections should be filtered out. This should depend on how bad false detections are to your system.
     // minNeighbors=2 means lots of good+bad detections, and minNeighbors=6 means only good detections are given but some are missed.
     int minNeighbors = 4;
-    printf("scaledWidth = %d\n",scaledWidth);
+    //printf("scaledWidth = %d\n",scaledWidth);
     // Perform Object or Face Detection, looking for just 1 object (the biggest in the image).
     vector<Rect> objects;
     detectObjectsCustom(img, cascade, objects, scaledWidth, flags, minFeatureSize, searchScaleFactor, minNeighbors);

face_recogition/main.cpp

@@ -73,7 +73,7 @@ const bool preprocessLeftAndRightSeparately = true;   // Preprocess left & right
 // Set to true if you want to see many windows created, showing various debug info. Set to 0 otherwise.
 bool m_debug = false;
 
-
+#include <signal.h>
 #include <stdio.h>
 #include <vector>
 #include <string>
@@ -98,9 +98,10 @@ using namespace std;
 
 #if !defined VK_ESCAPE
     #define VK_ESCAPE 0x1B      // Escape character (27)
+    #define VK_TAB 0x9
 #endif
 
-
+int server_stop = 0;
 // Running mode for the Webcam-based interactive GUI program.
 enum MODES {MODE_STARTUP=0, MODE_DETECTION, MODE_COLLECT_FACES, MODE_TRAINING, MODE_RECOGNITION, MODE_DELETE_ALL,   MODE_END};
 const char* MODE_NAMES[] = {"Startup", "Detection", "Collect Faces", "Training", "Recognition", "Delete All", "ERROR!"};
@@ -332,7 +333,7 @@ void recognizeAndTrainUsingWebcam(VideoCapture &videoCapture, CascadeClassifier
     // Since we have already initialized everything, lets start in Detection mode.
     m_mode = MODE_DETECTION;
 //dlx mod here First,a demo receive mat from zedboard use socket
-
+/*
       int sockfd, newsockfd, portno;
      socklen_t clilen;
      char buffer[1024];
@@ -413,7 +414,7 @@ void recognizeAndTrainUsingWebcam(VideoCapture &videoCapture, CascadeClassifier
 
      //waitKey(0);
 
-
+*/
 //
 
     // Run forever, until the user hits Escape to "break" out of this loop.
@@ -715,14 +716,173 @@ void recognizeAndTrainUsingWebcam(VideoCapture &videoCapture, CascadeClassifier
         if (keypress == VK_ESCAPE) {   // Escape Key
             // Quit the program!
             break;
+        }else if(keypress == VK_TAB){
+            //begin the server
+            if (model.empty())
+            {
+                model = loadModel();
+                //vector<int> getLabelsByString("labels");
+                faceLabels = model->getLabelsByString("labels");
+            }
+
+
+            int sockfd, newsockfd, portno;
+           socklen_t clilen;
+           char buffer[1024]="welcome";
+           char serverWelcome[10] = "welcome";
+           char serverBye[5] = "bye";
+           struct sockaddr_in serv_addr, cli_addr;
+           int n;
+
+           //printf("sizeof int %d\n",sizeof(int));
+           sockfd = socket(AF_INET, SOCK_STREAM, 0);
+           if (sockfd < 0)
+              perror("ERROR opening socket");
+           bzero((char *) &serv_addr, sizeof(serv_addr));
+           portno = 9001;
+           serv_addr.sin_family = AF_INET;
+           serv_addr.sin_addr.s_addr = INADDR_ANY;
+           serv_addr.sin_port = htons(portno);
+           if (bind(sockfd, (struct sockaddr *) &serv_addr,
+                    sizeof(serv_addr)) < 0)
+                    perror("ERROR on binding");
+
+           namedWindow( "Server", CV_WINDOW_AUTOSIZE );// Create a window for display.
+           while(server_stop != 1)
+           {
+             printf("listen=====\n");
+             listen(sockfd,5);
+             //printf("listen=----\n");
+             clilen = sizeof(cli_addr);
+             //printf("accepting=====\n");
+             newsockfd = accept(sockfd,
+                         (struct sockaddr *) &cli_addr,
+                         &clilen);
+             printf("accepting-----\n");
+             if (newsockfd < 0)
+                perror("ERROR on accept");
+
+             n = send(newsockfd,serverWelcome,strlen(serverWelcome),0);
+             printf("sebd n =%d\n",n);
+
+             bzero(buffer,1024);
+
+
+             int rows = 0,cols = 0,total = 0;
+             n = recv(newsockfd,buffer,43,0);
+             if (n < 0) printf("ERROR reading from socket\n");
+             //client printf("rows=%d , cols=%d , image.total()=%d \n",image.rows,image.cols,image.total());
+             sscanf(buffer,"rows=%d , cols=%d , image.total()=%d \n",&rows,&cols,&total);
+             printf(buffer);
+             printf("get it,rows=%d , cols=%d , image.total()=%d \n",rows,cols,total);
+             printf("\n");
+
+             Mat  img = Mat::zeros( rows,cols, CV_8UC3);
+             int  imgSize = img.total()*img.elemSize();
+             uchar sockData[imgSize];
+
+             for (int i = 0; i < imgSize; i += n) {
+                 if ((n = recv(newsockfd, sockData +i, imgSize  - i, 0)) == -1) {
+                   perror("recv failed");
+                 }
+                 printf("rcv n =%d\n",n);
+                 if(n == 0)
+                 {
+                    waitKey(0);
+                 }
+             }
+             n = send(newsockfd,serverBye,strlen(serverBye),0);
+             printf("sebd n =%d\n",n);
+             close(newsockfd);
+
+              int ptr=0;
+              for (int i = 0;  i < img.rows; i++) {
+                  for (int j = 0; j < img.cols; j++) {
+                      img.at<cv::Vec3b>(i,j) = cv::Vec3b(sockData[ptr+ 0],sockData[ptr+1],sockData[ptr+2]);
+                      ptr=ptr+3;
+                  }
+              }
+
+              printf("receive test ----rows =%d , cols = %d ,image.total() = %d \n",img.rows,img.cols,img.total());
+              imshow("Server", img);
+              //recognize the face
+
+              Rect faceRect;  // Position of detected face.
+              Rect searchedLeftEye, searchedRightEye; // top-left and top-right regions of the face, where eyes were searched.
+              Point leftEye, rightEye;    // Position of the detected eyes.
+
+              Mat preprocessedFace = getPreprocessedFaceFromClient(img, faceWidth, faceCascade, eyeCascade1, eyeCascade2, preprocessLeftAndRightSeparately, &faceRect, &leftEye, &rightEye, &searchedLeftEye, &searchedRightEye);
+
+              //printMatInfo(preprocessedFace,"preprocessedFace---");
+              bool gotFaceAndEyes = false;
+              if (preprocessedFace.data)
+                  gotFaceAndEyes = true;
+              //std::cout << "preprocessedFaces.size()" << preprocessedFaces.size() << std::endl;
+              //std::cout << "faceLabels.size()"<< faceLabels.size() << std::endl;
+              std::cout << "gotFaceAndEyes" <<  gotFaceAndEyes << std::endl;
+              //----recognize
+              //if (gotFaceAndEyes && (preprocessedFaces.size() > 0) && (preprocessedFaces.size() == faceLabels.size())) {
+              if(gotFaceAndEyes){
+                  // Generate a face approximation by back-projecting the eigenvectors & eigenvalues.
+                  Mat reconstructedFace;
+
+                  reconstructedFace = reconstructFace(model, preprocessedFace);
+                  if (1)
+                      if (reconstructedFace.data){
+                          imshow("reconstructedFace", reconstructedFace);
+                          imshow("Server",preprocessedFace);
+                          //waitKey(0);
+                      }
+                  // Verify whether the reconstructed face looks like the preprocessed face, otherwise it is probably an unknown person.
+                  double similarity = getSimilarity(preprocessedFace, reconstructedFace);
+                   cout << ". Similarity: " << similarity << endl;
+                  string outputStr;
+                  if (similarity < UNKNOWN_PERSON_THRESHOLD) {
+                      // Identify who the person is in the preprocessed face image.
+                      std::cout << "test1" << std::endl;
+                      identity = model->predict(preprocessedFace);
+                      std::cout << "test2" << std::endl;
+                      outputStr = toString(identity);
+                      std::cout << "test3" << std::endl;
+                  }
+                  else {
+                      // Since the confidence is low, assume it is an unknown person.
+                      outputStr = "Unknown";
+                  }
+
+                  //printf( "recong time = %gms\n", t/((double)cvGetTickFrequency()*1000.) );
+
+                  cout << "Identity: " << outputStr << ". Similarity: " << similarity << endl;
+
+              }
+
+
+              //waitKey(0);
+              //waitKey(0);
+          }
+           //Mat image(690,690,CV_8UC3,*buffer);
+           //imwrite("/home/securitas/images/prova.jpg",image);
+
+           close(sockfd);
+
+           //imshow( "Server", img );
+           //waitKey(0);
+           //return;
         }
 
+
     }//end while
 }
 
+void handle_sigtstp(int s) {//处理ctrz的
+    printf("DLX-------REVEIVED ctrz\n");
+    server_stop = 1;
+}
 
 int main(int argc, char *argv[])
 {
+    signal(SIGTSTP,handle_sigtstp);
+
     CascadeClassifier faceCascade;
     CascadeClassifier eyeCascade1;
     CascadeClassifier eyeCascade2;

face_recogition/preprocessFace.cpp

@@ -344,3 +344,121 @@ Mat getPreprocessedFace(Mat &srcImg, int desiredFaceWidth, CascadeClassifier &fa
     }
     return Mat();
 }
+
+Mat getPreprocessedFaceFromClient(Mat &srcImg, int desiredFaceWidth, CascadeClassifier &faceCascade, CascadeClassifier &eyeCascade1, CascadeClassifier &eyeCascade2, bool doLeftAndRightSeparately, Rect *storeFaceRect, Point *storeLeftEye, Point *storeRightEye, Rect *searchedLeftEye, Rect *searchedRightEye)
+{
+    // Use square faces.
+    int desiredFaceHeight = desiredFaceWidth;
+
+    Mat faceImg = srcImg;    // Get the detected face image.
+
+    // If the input image is not grayscale, then convert the BGR or BGRA color image to grayscale.
+    Mat gray;
+    if (faceImg.channels() == 3) {
+        cvtColor(faceImg, gray, CV_BGR2GRAY);
+    }
+    else if (faceImg.channels() == 4) {
+        cvtColor(faceImg, gray, CV_BGRA2GRAY);
+    }
+    else {
+        // Access the input image directly, since it is already grayscale.
+        gray = faceImg;
+    }
+
+    // Search for the 2 eyes at the full resolution, since eye detection needs max resolution possible!
+    Point leftEye, rightEye;
+
+    detectBothEyes(gray, eyeCascade1, eyeCascade2, leftEye, rightEye, searchedLeftEye, searchedRightEye);
+    //printf( "eye time = %gms\n", t/((double)cvGetTickFrequency()*1000.) );
+
+
+    // Give the eye results to the caller if desired.
+    if (storeLeftEye)
+        *storeLeftEye = leftEye;
+    if (storeRightEye)
+        *storeRightEye = rightEye;
+    std::cout << leftEye.x << rightEye.x << std::endl;
+    // Check if both eyes were detected.
+    if (leftEye.x >= 0 && rightEye.x >= 0) {
+
+        // Make the face image the same size as the training images.
+
+        // Since we found both eyes, lets rotate & scale & translate the face so that the 2 eyes
+        // line up perfectly with ideal eye positions. This makes sure that eyes will be horizontal,
+        // and not too far left or right of the face, etc.
+
+        // Get the center between the 2 eyes.
+        Point2f eyesCenter = Point2f( (leftEye.x + rightEye.x) * 0.5f, (leftEye.y + rightEye.y) * 0.5f );
+        // Get the angle between the 2 eyes.
+        double dy = (rightEye.y - leftEye.y);
+        double dx = (rightEye.x - leftEye.x);
+        double len = sqrt(dx*dx + dy*dy);
+        double angle = atan2(dy, dx) * 180.0/CV_PI; // Convert from radians to degrees.
+
+        // Hand measurements shown that the left eye center should ideally be at roughly (0.19, 0.14) of a scaled face image.
+        const double DESIRED_RIGHT_EYE_X = (1.0f - DESIRED_LEFT_EYE_X);
+        // Get the amount we need to scale the image to be the desired fixed size we want.
+        double desiredLen = (DESIRED_RIGHT_EYE_X - DESIRED_LEFT_EYE_X) * desiredFaceWidth;
+        double scale = desiredLen / len;
+        // Get the transformation matrix for rotating and scaling the face to the desired angle & size.
+        Mat rot_mat = getRotationMatrix2D(eyesCenter, angle, scale);
+        // Shift the center of the eyes to be the desired center between the eyes.
+        rot_mat.at<double>(0, 2) += desiredFaceWidth * 0.5f - eyesCenter.x;
+        rot_mat.at<double>(1, 2) += desiredFaceHeight * DESIRED_LEFT_EYE_Y - eyesCenter.y;
+
+        // Rotate and scale and translate the image to the desired angle & size & position!
+        // Note that we use 'w' for the height instead of 'h', because the input face has 1:1 aspect ratio.
+        Mat warped = Mat(desiredFaceHeight, desiredFaceWidth, CV_8U, Scalar(128)); // Clear the output image to a default grey.
+        warpAffine(gray, warped, rot_mat, warped.size());
+        //imshow("warped", warped);
+
+        // Give the image a standard brightness and contrast, in case it was too dark or had low contrast.
+        if (!doLeftAndRightSeparately) {
+            // Do it on the whole face.
+            equalizeHist(warped, warped);
+        }
+        else {
+            // Do it seperately for the left and right sides of the face.
+            equalizeLeftAndRightHalves(warped);
+        }
+        //imshow("equalized", warped);
+
+        // Use the "Bilateral Filter" to reduce pixel noise by smoothing the image, but keeping the sharp edges in the face.
+        Mat filtered = Mat(warped.size(), CV_8U);
+        bilateralFilter(warped, filtered, 0, 20.0, 2.0);
+        //imshow("filtered", filtered);
+
+        // Filter out the corners of the face, since we mainly just care about the middle parts.
+        // Draw a filled ellipse in the middle of the face-sized image.
+        Mat mask = Mat(warped.size(), CV_8U, Scalar(0)); // Start with an empty mask.
+        Point faceCenter = Point( desiredFaceWidth/2, cvRound(desiredFaceHeight * FACE_ELLIPSE_CY) );
+        Size size = Size( cvRound(desiredFaceWidth * FACE_ELLIPSE_W), cvRound(desiredFaceHeight * FACE_ELLIPSE_H) );
+        ellipse(mask, faceCenter, size, 0, 0, 360, Scalar(255), CV_FILLED);
+        //imshow("mask", mask);
+
+        // Use the mask, to remove outside pixels.
+        Mat dstImg = Mat(warped.size(), CV_8U, Scalar(128)); // Clear the output image to a default gray.
+        /*
+        namedWindow("filtered");
+        imshow("filtered", filtered);
+        namedWindow("dstImg");
+        imshow("dstImg", dstImg);
+        namedWindow("mask");
+        imshow("mask", mask);
+        */
+        // Apply the elliptical mask on the face.
+        filtered.copyTo(dstImg, mask);  // Copies non-masked pixels from filtered to dstImg.
+        //imshow("dstImg", dstImg);
+
+        return dstImg;
+    }
+    /*
+    else {
+        // Since no eyes were found, just do a generic image resize.
+        resize(gray, tmpImg, Size(w,h));
+    }
+    */
+    std::cout << "do not detecte eyes " << std::endl;
+
+    return Mat();
+}

face_recogition/preprocessFace.h

@@ -59,4 +59,4 @@ void equalizeLeftAndRightHalves(Mat &faceImg);
 // If a face is found, it can store the rect coordinates into 'storeFaceRect' and 'storeLeftEye' & 'storeRightEye' if given,
 // and eye search regions into 'searchedLeftEye' & 'searchedRightEye' if given.
 Mat getPreprocessedFace(Mat &srcImg, int desiredFaceWidth, CascadeClassifier &faceCascade, CascadeClassifier &eyeCascade1, CascadeClassifier &eyeCascade2, bool doLeftAndRightSeparately, Rect *storeFaceRect = NULL, Point *storeLeftEye = NULL, Point *storeRightEye = NULL, Rect *searchedLeftEye = NULL, Rect *searchedRightEye = NULL);
-
+Mat getPreprocessedFaceFromClient(Mat &srcImg, int desiredFaceWidth, CascadeClassifier &faceCascade, CascadeClassifier &eyeCascade1, CascadeClassifier &eyeCascade2, bool doLeftAndRightSeparately, Rect *storeFaceRect = NULL, Point *storeLeftEye = NULL, Point *storeRightEye = NULL, Rect *searchedLeftEye = NULL, Rect *searchedRightEye = NULL);

face_recogition/recognition.cpp

@@ -50,10 +50,38 @@ Ptr<FaceRecognizer> learnCollectedFaces(const vector<Mat> preprocessedFaces, con
 
     // Do the actual training from the collected faces. Might take several seconds or minutes depending on input!
     model->train(preprocessedFaces, faceLabels);
+    model->save("trainedModel.yml");
 
     return model;
 }
+Ptr<FaceRecognizer> loadModel()
+{
+    Ptr<FaceRecognizer> model;
+
+    bool haveContribModule = initModule_contrib();
+    if (!haveContribModule) {
+        cerr << "ERROR: The 'contrib' module is needed for FaceRecognizer but has not been loaded into OpenCV!" << endl;
+        exit(1);
+    }
 
+    model = Algorithm::create<FaceRecognizer>("FaceRecognizer.Fisherfaces");
+    Mat labels;
+    if (model.empty()) {
+        cerr << "ERROR: The FaceRecognizer algorithm is not available in your version of OpenCV. Please update to OpenCV v2.4.1 or newer." << endl;
+        exit(1);
+    }
+    try{
+      model->load("trainedModel.yml");
+      labels = model->get<Mat>("labels");
+    }catch (cv::Exception &e){}
+
+    if (labels.rows <= 0){
+       cerr << "ERROR: could not load trained data from trainedModel.yml" << endl;
+       exit(1);
+    }
+
+    return model;
+}
 // Convert the matrix row or column (float matrix) to a rectangular 8-bit image that can be displayed or saved.
 // Scales the values to be between 0 to 255.
 Mat getImageFrom1DFloatMat(const Mat matrixRow, int height)

face_recogition/recognition.h

@@ -38,7 +38,7 @@ using namespace std;
 //    "FaceRecognizer.Fisherfaces": Fisherfaces, also referred to as LDA (Belhumeur et al, 1997).
 //    "FaceRecognizer.LBPH":        Local Binary Pattern Histograms (Ahonen et al, 2006).
 Ptr<FaceRecognizer> learnCollectedFaces(const vector<Mat> preprocessedFaces, const vector<int> faceLabels, const string facerecAlgorithm = "FaceRecognizer.Eigenfaces");
-
+Ptr<FaceRecognizer> loadModel();
 // Show the internal face recognition data, to help debugging.
 void showTrainingDebugData(const Ptr<FaceRecognizer> model, const int faceWidth, const int faceHeight);