msprep returns incomplete object if a single patient still needs to make multiple transitions after the others are done

[evakoster]

In one of my datasets, I have one patient who makes far more transitions than the other patients. Msprep ignores most of these transitions and returns an incomplete msdata object, without giving a warning or error. It seems to stop as soon as only 1 patient is at risk for another transition.

[edbonneville]

Adding the reprex for this issue using the current Github version of {mstate}, since a similar issue was solved in #31 (but that fix does not seem to help here):

library(mstate)
#> Loading required package: survival

# code from Wreede et al. 2011
data("ebmt4")
ebmt <- ebmt4
tmat <- transMat(x = list(c(2, 3, 5, 6), c(4, 5, 6), c(4, 5, 6), c(5, 6), c(), c()), 
                 names = c("Tx", "Rec", "AE", "Rec+AE", "Rel", "Death"))

# own code
paths(tmat) # max 3 steps through the model
#>       [,1] [,2] [,3] [,4]
#>  [1,]    1   NA   NA   NA
#>  [2,]    1    2   NA   NA
#>  [3,]    1    2    4   NA
#>  [4,]    1    2    4    5
#>  [5,]    1    2    4    6
#>  [6,]    1    2    5   NA
#>  [7,]    1    2    6   NA
#>  [8,]    1    3   NA   NA
#>  [9,]    1    3    4   NA
#> [10,]    1    3    4    5
#> [11,]    1    3    4    6
#> [12,]    1    3    5   NA
#> [13,]    1    3    6   NA
#> [14,]    1    5   NA   NA
#> [15,]    1    6   NA   NA

#### add 1 more step which is made by only 1 patient (this still works) ####
# in this example, there are still more than 1 patient in the dataset after 3 steps (i.e., in a non-absorbing state after 3 steps)
# ebmt$id[ebmt$srv.s == 1 & ebmt$rec.s == 1 & ebmt$recae.s == 1] # all patients with 3 steps

# extend model with death after relapse and let only 1 patient die after relapse (the 4th step in the model)
ebmt$id[ebmt$srv.s == 1 & ebmt$rec.s == 1 & ebmt$recae.s == 1 & ebmt$rel.s == 1] 
#>  [1]    2    4   67   81  191  198  206  223  239  243  258  296  297  301  303
#> [16]  304  324  406  548  549  554  556  561  637  748  782  793  796  798  802
#> [31]  803  805  808  809  813  829  848  854  858  897  945  976  985 1016 1017
#> [46] 1078 1185 1192 1206 1224 1229 1280 1374 1390 1491 1531 1565 1585 1590 1625
#> [61] 1764 1773 1776 1777 1791 1799 1833 1845 1849 1850 1851 1925 1933 1936 1937
#> [76] 1954 1965 1985 1990 2012 2020 2024 2026 2058 2117 2121 2149 2153 2156 2158
#> [91] 2171 2177 2183 2188 2257
# id 2 is one of the patients who already made 3 steps
ebmt2 <- ebmt
ebmt2$srv.s = ifelse(ebmt2$id == 2, 1, ifelse(ebmt2$rel.s == 1, 0, ebmt2$srv.s))
# new transition matrix
tmat2 <- transMat(x = list(c(2, 3, 5, 6), c(4, 5, 6), c(4, 5, 6), c(5, 6), c(6), c()), 
                  names = c("Tx", "Rec", "AE", "Rec+AE", "Rel", "Death"))
paths(tmat2) 
#>       [,1] [,2] [,3] [,4] [,5]
#>  [1,]    1   NA   NA   NA   NA
#>  [2,]    1    2   NA   NA   NA
#>  [3,]    1    2    4   NA   NA
#>  [4,]    1    2    4    5   NA
#>  [5,]    1    2    4    5    6
#>  [6,]    1    2    4    6   NA
#>  [7,]    1    2    5   NA   NA
#>  [8,]    1    2    5    6   NA
#>  [9,]    1    2    6   NA   NA
#> [10,]    1    3   NA   NA   NA
#> [11,]    1    3    4   NA   NA
#> [12,]    1    3    4    5   NA
#> [13,]    1    3    4    5    6
#> [14,]    1    3    4    6   NA
#> [15,]    1    3    5   NA   NA
#> [16,]    1    3    5    6   NA
#> [17,]    1    3    6   NA   NA
#> [18,]    1    5   NA   NA   NA
#> [19,]    1    5    6   NA   NA
#> [20,]    1    6   NA   NA   NA
# check function
ms <- msprep(data = ebmt2, trans = tmat2, time = c(NA, "rec", "ae", "recae", "rel", "srv"), 
             status = c(NA, "rec.s", "ae.s", "recae.s", "rel.s", "srv.s"))
ms[ms$id == 2,] # still works (>1 persons in a non-absorbing state after 3 steps, even though only 1 makes a fourth transition)
#> An object of class 'msdata'
#> 
#> Data:
#>    id from to trans Tstart Tstop time status
#> 8   2    1  2     1      0    12   12      0
#> 9   2    1  3     2      0    12   12      1
#> 10  2    1  5     3      0    12   12      0
#> 11  2    1  6     4      0    12   12      0
#> 12  2    3  4     8     12    29   17      1
#> 13  2    3  5     9     12    29   17      0
#> 14  2    3  6    10     12    29   17      0
#> 15  2    4  5    11     29   422  393      1
#> 16  2    4  6    12     29   422  393      0
#> 17  2    5  6    13    422   579  157      1
# so, no problem here

#### add another step, only used by id = 2 ####
# in this example, for the fifth step we only have 1 patient left in the dataset
ebmt2$rel2 = ifelse(ebmt2$id == 2, ebmt2$rel + 1, ebmt2$srv)
ebmt2$rel2.s = ifelse(ebmt2$id == 2, 1, 0)
tmat3 <- transMat(x = list(c(2, 3, 5, 6), c(4, 5, 6), c(4, 5, 6), c(5, 6), c(6, 7), c(), c(6)), 
                  names = c("Tx", "Rec", "AE", "Rec+AE", "Rel", "Death", "Rel2"))
paths(tmat3)
#>       [,1] [,2] [,3] [,4] [,5] [,6]
#>  [1,]    1   NA   NA   NA   NA   NA
#>  [2,]    1    2   NA   NA   NA   NA
#>  [3,]    1    2    4   NA   NA   NA
#>  [4,]    1    2    4    5   NA   NA
#>  [5,]    1    2    4    5    6   NA
#>  [6,]    1    2    4    5    7   NA
#>  [7,]    1    2    4    5    7    6
#>  [8,]    1    2    4    6   NA   NA
#>  [9,]    1    2    5   NA   NA   NA
#> [10,]    1    2    5    6   NA   NA
#> [11,]    1    2    5    7   NA   NA
#> [12,]    1    2    5    7    6   NA
#> [13,]    1    2    6   NA   NA   NA
#> [14,]    1    3   NA   NA   NA   NA
#> [15,]    1    3    4   NA   NA   NA
#> [16,]    1    3    4    5   NA   NA
#> [17,]    1    3    4    5    6   NA
#> [18,]    1    3    4    5    7   NA
#> [19,]    1    3    4    5    7    6
#> [20,]    1    3    4    6   NA   NA
#> [21,]    1    3    5   NA   NA   NA
#> [22,]    1    3    5    6   NA   NA
#> [23,]    1    3    5    7   NA   NA
#> [24,]    1    3    5    7    6   NA
#> [25,]    1    3    6   NA   NA   NA
#> [26,]    1    5   NA   NA   NA   NA
#> [27,]    1    5    6   NA   NA   NA
#> [28,]    1    5    7   NA   NA   NA
#> [29,]    1    5    7    6   NA   NA
#> [30,]    1    6   NA   NA   NA   NA
ms2 <- msprep(data = ebmt2, trans = tmat3, time = c(NA, "rec", "ae", "recae", "rel", "srv", "rel2"), 
              status = c(NA, "rec.s", "ae.s", "recae.s", "rel.s", "srv.s", "rel2.s"))
ms2[ms2$id == 2,] # the last step is missing (id 2 should go from state 7 to state 6)
#> An object of class 'msdata'
#> 
#> Data:
#>    id from to trans Tstart Tstop time status
#> 8   2    1  2     1      0    12   12      0
#> 9   2    1  3     2      0    12   12      1
#> 10  2    1  5     3      0    12   12      0
#> 11  2    1  6     4      0    12   12      0
#> 12  2    3  4     8     12    29   17      1
#> 13  2    3  5     9     12    29   17      0
#> 14  2    3  6    10     12    29   17      0
#> 15  2    4  5    11     29   422  393      1
#> 16  2    4  6    12     29   422  393      0
#> 17  2    5  6    13    422   423    1      0
#> 18  2    5  7    14    422   423    1      1

#### check whether data is OK ####
library(dplyr)
#> 
#> Attaching package: 'dplyr'
#> The following objects are masked from 'package:stats':
#> 
#>     filter, lag
#> The following objects are masked from 'package:base':
#> 
#>     intersect, setdiff, setequal, union
library(tidyr)
ebmt2 %>% filter(id == 2) %>% select(rec:srv.s,rel2:rel2.s) %>% print() %>% 
  select(-ends_with(".s")) %>% pivot_longer(everything()) %>% arrange(value)
#>   rec rec.s ae ae.s recae recae.s rel rel.s srv srv.s rel2 rel2.s
#> 2  29     1 12    1    29       1 422     1 579     1  423      1
#> # A tibble: 6 × 2
#>   name  value
#>   <chr> <dbl>
#> 1 ae       12
#> 2 rec      29
#> 3 recae    29
#> 4 rel     422
#> 5 rel2    423
#> 6 srv     579
# data for id 2 is fine

devtools::session_info()
#> ─ Session info ───────────────────────────────────────────────────────────────
#>  setting  value
#>  version  R version 4.3.3 (2024-02-29)
#>  os       Linux Mint 22.1
#>  system   x86_64, linux-gnu
#>  ui       X11
#>  language en_GB
#>  collate  en_GB.UTF-8
#>  ctype    en_GB.UTF-8
#>  tz       Europe/Berlin
#>  date     2025-07-14
#>  pandoc   3.4 @ /usr/lib/rstudio/resources/app/bin/quarto/bin/tools/x86_64/ (via rmarkdown)
#>  quarto   1.6.42 @ /usr/lib/rstudio/resources/app/bin/quarto/bin/quarto
#> 
#> ─ Packages ───────────────────────────────────────────────────────────────────
#>  package     * version date (UTC) lib source
#>  cachem        1.1.0   2024-05-16 [1] CRAN (R 4.3.3)
#>  cli           3.6.5   2025-04-23 [1] CRAN (R 4.3.3)
#>  data.table    1.17.6  2025-06-17 [1] CRAN (R 4.3.3)
#>  devtools      2.4.5   2022-10-11 [1] CRAN (R 4.3.3)
#>  digest        0.6.37  2024-08-19 [1] CRAN (R 4.3.3)
#>  dplyr       * 1.1.4   2023-11-17 [1] CRAN (R 4.3.3)
#>  ellipsis      0.3.2   2021-04-29 [1] CRAN (R 4.3.3)
#>  evaluate      1.0.4   2025-06-18 [1] CRAN (R 4.3.3)
#>  fastmap       1.2.0   2024-05-15 [1] CRAN (R 4.3.3)
#>  fs            1.6.6   2025-04-12 [1] CRAN (R 4.3.3)
#>  generics      0.1.4   2025-05-09 [1] CRAN (R 4.3.3)
#>  glue          1.8.0   2024-09-30 [1] CRAN (R 4.3.3)
#>  htmltools     0.5.8.1 2024-04-04 [1] CRAN (R 4.3.3)
#>  htmlwidgets   1.6.4   2023-12-06 [1] CRAN (R 4.3.3)
#>  httpuv        1.6.16  2025-04-16 [1] CRAN (R 4.3.3)
#>  knitr         1.50    2025-03-16 [1] CRAN (R 4.3.3)
#>  later         1.4.2   2025-04-08 [1] CRAN (R 4.3.3)
#>  lattice       0.22-5  2023-10-24 [4] CRAN (R 4.3.3)
#>  lifecycle     1.0.4   2023-11-07 [1] CRAN (R 4.3.3)
#>  magrittr      2.0.3   2022-03-30 [1] CRAN (R 4.3.3)
#>  Matrix        1.6-5   2024-01-11 [4] CRAN (R 4.3.2)
#>  memoise       2.0.1   2021-11-26 [1] CRAN (R 4.3.3)
#>  mime          0.13    2025-03-17 [1] CRAN (R 4.3.3)
#>  miniUI        0.1.2   2025-04-17 [1] CRAN (R 4.3.3)
#>  mstate      * 0.3.3   2025-07-14 [1] Github (hputter/mstate@3ca3c5f)
#>  pillar        1.11.0  2025-07-04 [1] CRAN (R 4.3.3)
#>  pkgbuild      1.4.8   2025-05-26 [1] CRAN (R 4.3.3)
#>  pkgconfig     2.0.3   2019-09-22 [1] CRAN (R 4.3.3)
#>  pkgload       1.4.0   2024-06-28 [1] CRAN (R 4.3.3)
#>  profvis       0.4.0   2024-09-20 [1] CRAN (R 4.3.3)
#>  promises      1.3.3   2025-05-29 [1] CRAN (R 4.3.3)
#>  purrr         1.0.4   2025-02-05 [1] CRAN (R 4.3.3)
#>  R6            2.6.1   2025-02-15 [1] CRAN (R 4.3.3)
#>  Rcpp          1.1.0   2025-07-02 [1] CRAN (R 4.3.3)
#>  remotes       2.5.0   2024-03-17 [1] CRAN (R 4.3.3)
#>  reprex        2.1.1   2024-07-06 [1] CRAN (R 4.3.3)
#>  rlang         1.1.6   2025-04-11 [1] CRAN (R 4.3.3)
#>  rmarkdown     2.29    2024-11-04 [1] CRAN (R 4.3.3)
#>  rstudioapi    0.17.1  2024-10-22 [1] CRAN (R 4.3.3)
#>  sessioninfo   1.2.3   2025-02-05 [1] CRAN (R 4.3.3)
#>  shiny         1.11.1  2025-07-03 [1] CRAN (R 4.3.3)
#>  survival    * 3.5-8   2024-02-14 [4] CRAN (R 4.3.2)
#>  tibble        3.3.0   2025-06-08 [1] CRAN (R 4.3.3)
#>  tidyr       * 1.3.1   2024-01-24 [1] CRAN (R 4.3.3)
#>  tidyselect    1.2.1   2024-03-11 [1] CRAN (R 4.3.3)
#>  urlchecker    1.0.1   2021-11-30 [1] CRAN (R 4.3.3)
#>  usethis       3.1.0   2024-11-26 [1] CRAN (R 4.3.3)
#>  utf8          1.2.6   2025-06-08 [1] CRAN (R 4.3.3)
#>  vctrs         0.6.5   2023-12-01 [1] CRAN (R 4.3.3)
#>  withr         3.0.2   2024-10-28 [1] CRAN (R 4.3.3)
#>  xfun          0.52    2025-04-02 [1] CRAN (R 4.3.3)
#>  xtable        1.8-4   2019-04-21 [1] CRAN (R 4.3.3)
#>  yaml          2.3.10  2024-07-26 [1] CRAN (R 4.3.3)
#> 
#>  [1] /home/edbonn/R/x86_64-pc-linux-gnu-library/4.3
#>  [2] /usr/local/lib/R/site-library
#>  [3] /usr/lib/R/site-library
#>  [4] /usr/lib/R/library
#>  * ── Packages attached to the search path.
#> 
#> ──────────────────────────────────────────────────────────────────────────────

^{Created on 2025-07-14 with reprex v2.1.1}