# -----------------------------------------------------------------------------
# Program: runtime_test.R
# Author: Loki
# Date: 2019.08.12
#
# Purpose:
#      Comparing runtime of "MAVE" with "CVE".
#
# RevisionHistory:
#      Loki -- 2019.08.12 initial creation
# -----------------------------------------------------------------------------

# load CVE package
library(CVE)
# load MAVE package for comparison
library(MAVE)

# set nr of simulations per dataset
nr.sim <- 10

# set names of datasets to run tests on
dataset.names <- c("M1", "M2", "M3", "M4", "M5")

#' Orthogonal projection to sub-space spanned by `B`
#'
#' @param B Matrix
#' @return Orthogonal Projection Matrix
proj <- function(B) {
    B %*% solve(t(B) %*% B) %*% t(B)
}

#' Compute nObs given dataset dimension \code{n}.
#'
#' @param n Number of samples
#' @return Numeric estimate of \code{nObs}
nObs <- function (n) { n^0.5 }

## prepare "logging"
# result error, time, ... data.frame's
error <- matrix(nrow = nr.sim, ncol = 2 * length(dataset.names))
time <- matrix(nrow = nr.sim, ncol = 2 * length(dataset.names))
# convert to data.frames
error <- as.data.frame(error)
time <- as.data.frame(time)
# set names
names(error) <- kronecker(c("CVE.", "MAVE."), dataset.names, paste0)
names(time) <- kronecker(c("CVE.", "MAVE."), dataset.names, paste0)

# get current time
start.time <- Sys.time()
## main comparison loop (iterate `nr.sim` times for each dataset)
for (i in seq_along(dataset.names)) {
    for (j in 1:nr.sim) {
        name <- dataset.names[i]
        # reporting progress
        cat("\rRunning Test (", name, j , "):",
            (i - 1) * nr.sim + j, "/", length(dataset.names) * nr.sim,
            " - Time since start:", format(Sys.time() - start.time), "\033[K")
        # sample new dataset
        ds <- dataset(name)
        k <- ncol(ds$B) # real dim
        data <- data.frame(X = ds$X, Y = ds$Y)
        # call CVE
        cve.time <- system.time(
            cve.res <- cve(Y ~ .,
                           data = data,
                           method = "simple",
                           k = k)
        )
        # call MAVE
        mave.time <- system.time(
            mave.res <- mave(Y ~ .,
                             data = data,
                             method = "meanMAVE")
        )
        # compute real and approximated sub-space projections
        P <- proj(ds$B) # real
        P.cve <- proj(cve.res$B)
        P.mave <- proj(mave.res$dir[[k]])
        # compute (and store) errors
        error[j, paste0("CVE.", name)] <- norm(P - P.cve, 'F') / sqrt(2 * k)
        error[j, paste0("MAVE.", name)] <- norm(P - P.mave, 'F') / sqrt(2 * k)
        # store run-times
        time[j, paste0("CVE.", name)] <- cve.time["elapsed"]
        time[j, paste0("MAVE.", name)] <- mave.time["elapsed"]
    }
}

cat("\n\n## Time [sec] Means:\n")
print(colMeans(time))
cat("\n## Error Means:\n")
print(colMeans(error))

len <- length(dataset.names)
boxplot(as.matrix(error),
    main = paste0("Error (nr.sim = ", nr.sim, ")"),
    ylab = expression(error == group("||", P[B] - P[hat(B)], "||")[F] / sqrt(2*k)),
    las = 2,
    at = c(1:len, 1:len + len + 1)
)
boxplot(as.matrix(time),
    main = paste0("Time (nr.sim = ", nr.sim, ")"),
    ylab = "time [sec]",
    las = 2,
    at = c(1:len, 1:len + len + 1)
)
