test_flattenLong.R

df <- structure(list(CAT = structure(1:2, .Label = c("A", "B"), class = "factor"), 
    COUNT = list(1:3, 4:5), TREAT = list(c("Treat-a", "Treat-b"
), c("Treat-c", "Treat-d", "Treat-e"))), .Names = c("CAT", 
"COUNT", "TREAT"), row.names = c(NA, -2L), class = "data.frame")

flatten <- function(indt, cols, drop = FALSE) {
  require(data.table)
  if (!is.data.table(indt)) indt <- as.data.table(indt)
  x <- unlist(indt[, lapply(.SD, function(x) max(lengths(x))), .SDcols = cols])
  nams <- paste(rep(cols, x), sequence(x), sep = "_")
  indt[, (nams) := unlist(lapply(.SD, transpose), recursive = FALSE), .SDcols = (cols)]
  if (isTRUE(drop)) indt[, (cols) := NULL]
  indt[]
}

## Works as expected....
flatten(df, c("COUNT", "TREAT"), TRUE)
#    CAT COUNT_1 COUNT_2 COUNT_3 TREAT_1 TREAT_2 TREAT_3
# 1:   A       1       2       3 Treat-a Treat-b      NA
# 2:   B       4       5      NA Treat-c Treat-d Treat-e


flattenLong <- function(indt, cols) {
  ob <- setdiff(names(indt), cols)
  x <- flatten(indt, cols, TRUE)
  mv <- lapply(cols, function(y) grep(sprintf("^%s_", y), names(x)))
  setorderv(melt(x, measure.vars = mv, value.name = cols), ob)[]
}

flattenLong(df, c("COUNT", "TREAT"))
#    CAT variable COUNT   TREAT
# 1:   A        1     1 Treat-a
# 2:   A        2     2 Treat-b
# 3:   A        3     3      NA
# 4:   B        1     4 Treat-c
# 5:   B        2     5 Treat-d
# 6:   B        3    NA Treat-e

Possibility:

flattenLong <- function(indt, cols) {
  ob <- setdiff(names(indt), cols)
  x <- flatten(indt, cols, TRUE)
  mv <- lapply(cols, function(y) grep(sprintf("^%s_", y), names(x)))
  setorderv(melt(x, measure.vars = mv, value.name = cols), ob)[]
}