matlab_version/face_validation/Create_data_train.m

function Create_data_train()

load '../models/pdm/pdm_68_aligned_menpo';
load '../models/tri_68.mat';

% This script uses the same format used for patch expert training, and
% expects the data to be there (this can be found in
% https://github.com/TadasBaltrusaitis/CCNF)

% Replace with your location of training data
dataset_loc = 'C:\Users\tbaltrus\Documents\CCNF\patch_experts\data_preparation/prepared_data/';
addpath('../PDM_helpers/');
addpath('./paw_helpers/');

% Collect Menpo, Multi-PIE and 300W data for training the validator
scale = '0.5';
prefix_menpo= 'menpo_train_';
prefix_mpie_300W = 'combined_';

% Find the available positive training data
data_files = dir(sprintf('%s/%s%s*.mat', dataset_loc, prefix_menpo, scale));
data_files_c = dir(sprintf('%s/%s%s*.mat', dataset_loc, prefix_mpie_300W, scale));

centres_all = [];
for i=1:numel(data_files)
   
    % Load the orientation of the training data
    load([dataset_loc, '/', data_files(i).name], 'centres');
    centres_all = cat(1, centres_all, centres);
    
end

% Construct mirror indices (which views need to be flipped to create other
% profile training data)
mirror_inds = zeros(size(centres_all,1), 1);
for i=1:numel(data_files)
    
    % mirrored image has inverse yaw
    mirrored_centre = centres_all(i,:);
    mirrored_centre(2) = -mirrored_centre(2);

    % if mirrored version has same orientation, do not need mirroring
    if(~isequal(mirrored_centre, centres_all(i,:)))
       
        centres_all = cat(1, centres_all, mirrored_centre);
        mirror_inds = cat(1, mirror_inds, i);
        
    end    
    
end
        
% Replace with your location of training data
outputLocation = 'D:\Datasets/detection_validation/prep_data/';
    
num_more_neg = 10;

% Make sure same data generated all the time
rng(0);

% Negative samples from teh INRIAPerson dataset
neg_image_loc = 'D:\Datasets\INRIAPerson\INRIAPerson\Train\neg/';

neg_images = cat(1,dir([neg_image_loc, '/*.jpg']),dir([neg_image_loc, '/*.png']));

max_img_used = 8000;

% do it separately for centers due to memory limitations
for r=1:size(centres_all,1)
    
    a_mod = 0.4;
    
    mirror = false;
    
    if(mirror_inds(r) ~= 0 )
        mirror = true;
        label_mirror_inds = [1,17;2,16;3,15;4,14;5,13;6,12;7,11;8,10;18,27;19,26;20,25;21,24;22,23;...
                  32,36;33,35;37,46;38,45;39,44;40,43;41,48;42,47;49,55;50,54;51,53;60,56;59,57;...
                  61,65;62,64;68,66];    
        load([dataset_loc, '/', data_files_c(mirror_inds(r)).name]);
        all_images_t = all_images;
        landmark_locations_t = landmark_locations;
        visiIndex_t = visiIndex;

        load([dataset_loc, '/', data_files(mirror_inds(r)).name]);

        % Combining Menpo + MPIE + 300W
        all_images = cat(1, all_images, all_images_t);
        landmark_locations = cat(1, landmark_locations, landmark_locations_t);        
        % Taking a subset of visibilities from all the datasets
        visiIndex = visiIndex_t & visiIndex;
    else            
        load([dataset_loc, '/', data_files_c(r).name]);
        all_images_t = all_images;
        landmark_locations_t = landmark_locations;
        visiIndex_t = visiIndex;
        
        load([dataset_loc, '/', data_files(r).name]);
        all_images = cat(1, all_images, all_images_t);
        landmark_locations = cat(1, landmark_locations, landmark_locations_t);
        visiIndex = visiIndex_t & visiIndex;
    end
    
    visiCurrent = logical(visiIndex);
    
    if(mirror)
        centres = [centres(1), -centres(2), -centres(3)];
        tmp1 = visiCurrent(label_mirror_inds(:,1));
        tmp2 = visiCurrent(label_mirror_inds(:,2));            
        visiCurrent(label_mirror_inds(:,2)) = tmp1;
        visiCurrent(label_mirror_inds(:,1)) = tmp2; 
    end    
            
    visibleVerts = 1:numel(visiCurrent);
    visibleVerts = visibleVerts(visiCurrent)-1;

    % Correct the triangulation to take into account the vertex
    % visibilities
    triangulation = [];

    shape = a_mod * Euler2Rot(centres * pi/180) * reshape(M, numel(M)/3, 3)';
    shape = shape';

    for i=1:size(T,1)
        visib = 0;
        for j=1:numel(visibleVerts)
            if(T(i,1)==visibleVerts(j))
                visib = visib+1;
            end
            if(T(i,2)==visibleVerts(j))
                visib = visib+1;
            end
            if(T(i,3)==visibleVerts(j))
                visib = visib+1;
            end
        end
        
        % Only if all three of the vertices are visible
        if(visib == 3)            
            
            % Also want to remove triangles facing the wrong way (self occluded)
            v1 = [shape(T(i,1)+1,1), shape(T(i,1)+1,2), shape(T(i,1)+1,3)];
            v2 = [shape(T(i,2)+1,1), shape(T(i,2)+1,2), shape(T(i,2)+1,3)];
            v3 = [shape(T(i,3)+1,1), shape(T(i,3)+1,2), shape(T(i,3)+1,3)];
            normal = cross((v2-v1), v3 - v2);
            normal = normal / norm(normal);
            direction = normal * [0,0,1]';

            % And only if the triangle is facing the camera
            if(direction > 0)
                triangulation = cat(1, triangulation, T(i,:)); 
            end
        end                
    end
        
    % Initialise the warp
    [ alphas, betas, triX, mask, minX, minY, nPix ] = InitialisePieceWiseAffine(triangulation, shape);

    mask = logical(mask);
    
    imgs_to_use = randperm(size(landmark_locations, 1));
    
    if(size(landmark_locations, 1) > max_img_used)
        imgs_to_use = imgs_to_use(1:max_img_used);
    end
    
    % Extracting relevant filenames
    examples = zeros(numel(imgs_to_use) * (num_more_neg+1), nPix);    
    errors = zeros(numel(imgs_to_use) * (num_more_neg+1), 1);
    
    unused_pos = 0;
    
    curr_filled = 0;
           
    for j=imgs_to_use
        
        labels = squeeze(landmark_locations(j,:,:));

        img = squeeze(all_images(j,:,:));

        if(mirror)
            img = fliplr(img);
            imgSize = size(img);
            flippedLbls = labels;
            flippedLbls(:,1) = imgSize(1) - flippedLbls(:,1) + 1;
            tmp1 = flippedLbls(label_mirror_inds(:,1),:);
            tmp2 = flippedLbls(label_mirror_inds(:,2),:);            
            flippedLbls(label_mirror_inds(:,2),:) = tmp1;
            flippedLbls(label_mirror_inds(:,1),:) = tmp2;   
            labels = flippedLbls;
        end            
        
        % If for some reason some of the labels are not visible in the
        % current sample skip this label
        non_existent_labels = labels(:,1)==0 | labels(:,2)==0;
        non_existent_inds = find(non_existent_labels)-1;
        if(numel(intersect(triangulation(:), non_existent_inds)) > 0)
           unused_pos = unused_pos + 1; 
           continue;
        end
        
        % Centering the pixel so that 0,0 is center of the top left pixel
        labels = labels - 1;
        
        curr_filled = curr_filled + 1;        
        [features] = ExtractFaceFeatures(img, labels, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
%         sample_img = zeros(size(mask));sample_img(mask) = features;imagesc(sample_img)
        examples(curr_filled,:) = features;
        errors(curr_filled,:) = 0;
        
        % Extract the correct PDM parameters for the model (we will perturb
        % them for some negative examples)
        [ a_orig, R_orig, trans_orig, ~, params_orig] = fit_PDM_ortho_proj_to_2D(M, E, V, labels);
        eul_orig = Rot2Euler(R_orig);
        
        % a slightly perturbed example, too tight
        % from 0.3 to 0.9
        a_mod = a_orig * (0.6 + (randi(7) - 4)*0.1);                        
        p_global = [a_mod; eul_orig'; trans_orig];
        
        labels_mod = GetShapeOrtho(M, V, params_orig, p_global);
        labels_mod = labels_mod(:,1:2);
        
        [features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
%         sample_img = zeros(size(mask));sample_img(mask) = features;imagesc(sample_img)

        curr_filled = curr_filled + 1;        
        examples(curr_filled,:) = features;
        
        % Compute the badness of fit
        error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
        errors(curr_filled,:) = error;

        % a slightly perturbed example, too broad
        % from 1.2 to 0.6
        a_mod = a_orig * (1.4 + (randi(5) - 3)*0.1);        
        p_global = [a_mod; eul_orig'; trans_orig];
        
        labels_mod = GetShapeOrtho(M, V, params_orig, p_global);
        labels_mod = labels_mod(:,1:2);
        
        [features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
%         sample_img = zeros(size(mask));sample_img(mask) = features;imagesc(sample_img)
        
        curr_filled = curr_filled + 1;        
        examples(curr_filled,:) = features;
        
        error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
        errors(curr_filled,:) = error;

        % A somewhat offset example
        
        trans_mod = trans_orig + randn(2,1) * 20;
        p_global = [a_orig; eul_orig'; trans_mod];
        
        labels_mod = GetShapeOrtho(M, V, params_orig, p_global);
        labels_mod = labels_mod(:,1:2);
        
        [features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
        
        curr_filled = curr_filled + 1;        
        examples(curr_filled,:) = features;
    
        error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
        errors(curr_filled,:) = error;
       
        % A rotated sample
        eul_mod = eul_orig + randn(1,3)*0.3;
        p_global = [a_orig; eul_mod'; trans_orig];
        
        labels_mod = GetShapeOrtho(M, V, params_orig, p_global);
        labels_mod = labels_mod(:,1:2);
        
        [features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
        
        curr_filled = curr_filled + 1;        
        examples(curr_filled,:) = features;
    
        error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
        errors(curr_filled,:) = error;
        
        % A sample with modified shape parameters
        p_global = [a_orig; eul_orig'; trans_orig];
        params_mod = params_orig + randn(size(params_orig)).*sqrt(E);
        labels_mod = GetShapeOrtho(M, V, params_mod, p_global);
        labels_mod = labels_mod(:,1:2);
        
        [features] = ExtractFaceFeatures(img, labels_mod, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
        
        curr_filled = curr_filled + 1;        
        examples(curr_filled,:) = features;
    
        error = norm(labels_mod(:) - labels(:)) / (max(labels(:,2))-min(labels(:,2)));
        errors(curr_filled,:) = error;     
                
        % pick a random image from negative inriaperson dataset, use original location if
        % first, otherwhise resize it to fit
        for n=6:num_more_neg
            n_img = randi(numel(neg_images));
            
            neg_image = imread([neg_image_loc, neg_images(n_img).name]);
            
            if(size(neg_image,3) == 3)
                neg_image = rgb2gray(neg_image);
            end
            
            [h_neg, w_neg] = size(neg_image);
            
            % if the current labels fit just use them, if not, then resize
            % to fit
            max_x = max(labels(:,1));
            max_y = max(labels(:,2));
            
            if(max_x > w_neg || max_y > h_neg)
               neg_image = imresize(neg_image, [max_y, max_x]);
            end
            
            [features] = ExtractFaceFeatures(neg_image, labels, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
            
            curr_filled = curr_filled + 1;        
            examples(curr_filled,:) = features;

            % Set high error to 3
            errors(curr_filled,:) = 3;
        end
        
        
        if(mod(curr_filled, 10) == 0)
            fprintf('%d/%d done\n', curr_filled/(num_more_neg+1), numel(imgs_to_use));
        end
        % add the pos example to the background
        
    end    
                
    examples = examples(1:curr_filled,:);
    errors = errors(1:curr_filled);
    
    filename = sprintf('%s/face_validator_train_%d.mat', outputLocation, r);
    save(filename, 'examples', 'errors', 'alphas', 'betas', 'triangulation', 'minX', 'minY', 'nPix', 'shape', 'triX', 'mask', 'centres');
    
    
end

end

function [features] = ExtractFaceFeatures(img, labels, triangulation, triX, mask, alphas, betas, nPix, minX, minY)

    % Make sure labels are within range
    [hRes, wRes] = size(img);
    labels(labels(:,1) < 0,1) = 0;
    labels(labels(:,2) < 0,2) = 0;

    labels(labels(:,1) > wRes-1,1) = wRes-1;
    labels(labels(:,2) > hRes-1,2) = hRes-1;   

    crop_img = Crop(img, labels, triangulation, triX, mask, alphas, betas, nPix, minX, minY);
    crop_img(isnan(crop_img)) = 0;
    
    % vectorised version
    features = reshape(crop_img(logical(mask)), 1, nPix);

    % normalisations
    features = (features - mean(features));
    norms = std(features);
    if(norms==0)            
        norms = 1;
    end
    features = features / norms;

end