some test script for the standardize method

hurdle.jl

using Test, Distributions, CSV, GLM, Lasso, DataFrames
using Random, Distributed, LinearAlgebra, SparseArrays, SharedArrays

include("./test/testutils.jl")
include("./test/addworkers.jl")

import HurdleDMR; @everywhere using HurdleDMR

include("./test/testdata.jl")

# data init
testdir = "./test/";
bioChemists=CSV.read(joinpath(testdir,"data","bioChemists.csv"));
bioChemists[:marMarried]=bioChemists[:mar] .== "Married";
bioChemists[:femWomen]=bioChemists[:fem] .== "Women";
bioChemists[:art] = convert(Array{Union{Float64, Missings.Missing},1}, bioChemists[:art]);

X=convert(Array{Float64,2},bioChemists[[:femWomen,:marMarried,:kid5,:phd,:ment]]);
Xwconst=[ones(size(X,1)) X];
y=convert(Array{Float64,1},bioChemists[:art]);
const ixpartial = 50:60;

coefsR3=vec(convert(Matrix{Float64},CSV.read(joinpath(testdir,"data","hurdle_coefsR3.csv"))));
yhatR3=vec(convert(Matrix{Float64},CSV.read(joinpath(testdir,"data","hurdle_yhatR3.csv"))));
yhatR3partial=vec(convert(Matrix{Float64},CSV.read(joinpath(testdir,"data","hurdle_yhatR3partial.csv"))));

offpos = convert(Vector{Float64},bioChemists[:offpos]);
offzero = convert(Vector{Float64},bioChemists[:offzero]);

# some modification
_Xpos = X[:,1:3];
_Xzero = X[:,4:5];

X, norm = Lasso.standardizeX(X, true);

Xpos = X[:,1:3];
Xzero = X[:,4:5];
Xzerowconst=[ones(size(X,1)) Xzero];
Xposwconst=[ones(size(X,1)) Xpos];


# same simple hurdle through UNREGULATED lasso path
hurdleglrfit = fit(Hurdle,GammaLassoPath,Xzero,y; standardize=false, Xpos=Xpos, offsetzero=offzero, offsetpos=offpos, λ=[0.0,0.0001])

lmul!(Diagonal(norm[4:5]), hurdleglrfit.mzero.coefs)
lmul!(Diagonal(norm[1:3]), hurdleglrfit.mpos.coefs)

coefsJ=vcat(coef(hurdleglrfit;select=:AICc)...)
@test coefsJ ≈ coefsR3 rtol=1e-4
# rdist(coefsJ,coefsR3)
yhatJ=predict(hurdleglrfit, _Xzero; Xpos=_Xpos, offsetzero=offzero, offsetpos=offpos, select=:AICc)

@test yhatJ ≈ yhatR3 rtol=1e-4
yhatJpartial=predict(hurdleglrfit, _Xzero[ixpartial,:]; Xpos=_Xpos[ixpartial,:], offsetzero=offzero[ixpartial], offsetpos=offpos[ixpartial], select=:AICc)
@test yhatJpartial ≈ yhatR3partial rtol=1e-4
yhatJ = predict(hurdleglrfit, Xzero; Xpos=Xpos, offsetzero=offzero, offsetpos=offpos, select=:all)
@test size(yhatJ) == (size(X,1),2)