Dropout (inverted dropout)

ml_dropout_inverted_dropout.m

disp("Dropout, overcome the overfitting in NN....");

function [X] = dropout(X, keep_prob)
    % Dropout some units from X.
    % (1 - keep_prob) of units will be dropped out.

    sz = size(X);

    mask = rand(sz);
    mask = mask < 0.8; % Element of mask will be set to 1 or 0 with probability 𝑘𝑒𝑒𝑝_𝑝𝑟𝑜𝑏
    mask

    X = X .* mask;

    % The element size of X (input) has been reduced by keep_prob from mask (a percentage of elements have been dropped out by mask),
    % thus the value of X (output) is also gonna be reduced, so to compensate this roughly we shall invert the change by dividing
    % keep_prob to make sure the value of X (output) won't be impacted mostly.

    X = X ./ keep_prob;

endfunction

L = randi(35, 3, 4);
keep_prob = 0.85;

L
printf("sum(L): %f\n", sum(L, 2));
L = dropout(L, keep_prob);
printf("sum(L): %f\n", sum(L, 2));

% Output like:
% L =

% 33 8 33 18
% 6 27 32 24
% 23 26 27 31

% sum(L):92.000000
% sum(L):89.000000
% sum(L):107.000000
% mask =

% 1 1 1 1
% 1 0 1 1
% 1 1 1 0

% sum(L):108.235294
% sum(L):72.941176
% sum(L):89.411765