Simulations in SAS

Fallacy of a single experiment

data normal;
call streaminit(123);
 length simulation 4 i 3; 
 do simulation = 1 to 1000;
  do i = 1 to 100;
   group='A';
    x = rand('normal');
 output;
 group='B';
    x = rand('normal');
    output;
  end;
 end;
run;

ods _all_ close;
ods output ttests=ttests;
proc ttest plots=none data=normal;
 by simulation;
 class group;
 var x;
run;

data ttests;
set ttests;
if variances='Equal';
type_one_error = probt LE 0.05;
retain cumulative_error_count;
format cumulative_error_rate percent10.2;
label cumulative_error_rate = 'Cumulative error rate';
if simulation eq 1 then cumulative_error_count = 0;
cumulative_error_count+type_one_error;
cumulative_error_rate = cumulative_error_count /simulation;
run;

ods html;
proc freq data=ttests;
table type_one_error/nocum;
run;

ods graphics on / width=10in height=5in imagefmt=svg;
proc sgplot data=ttests;
series x=simulation y=cumulative_error_rate / smoothconnect;
refline 0.05 /axis=y lineattrs=(color=red) label="Type I error";
xaxis label="Number of Simulations";
yaxis grid;
title "Simulation of type I error (False Positive) in a T-Test";
run;