test: literature-grounded FIPC regression/robustness suite

.github/workflows/r.yml

@@ -16,7 +16,15 @@ concurrency:
 
 jobs:
   check:
-    runs-on: ubuntu-latest
+    # Pin to jammy (22.04): Posit P3M publishes precompiled Linux binaries for
+    # the current R release on jammy, but not yet for noble (24.04, the current
+    # ubuntu-latest). On noble, pak falls back to source-building the mirt ->
+    # SimDesign -> qs -> stringfish chain, and the source-built stringfish.so
+    # then fails to load with an Intel TBB ABI mismatch (undefined symbol
+    # tbb::internal::concurrent_vector_base_v3::internal_grow_by), aborting
+    # dependency install before R CMD check ever runs. Jammy binaries avoid the
+    # source compile entirely. Revisit once P3M ships noble binaries for these.
+    runs-on: ubuntu-22.04
     env:
       R_PROFILE_USER: /dev/null
 

docs/references.md

@@ -0,0 +1,55 @@
+<!-- markdownlint-configure-file { "MD013": { "line_length": 120 } } -->
+
+# References
+
+This file records, in APA 7th-edition form, the psychometric literature that
+grounds the regression and robustness tests under `tests/testthat/`. Each work
+was verified against its publisher record (journal, volume, issue, pages, and
+DOI) rather than reproduced from memory.
+
+> Note: a local Zotero API is not reachable from the build/CI sandbox, so this
+> in-repo, version-controlled reference list is the achievable substitute for a
+> Zotero-managed bibliography. Keep it in sync with the citation headers in the
+> test files.
+
+## Cited works
+
+- Bock, R. D., & Aitkin, M. (1981). Marginal maximum likelihood estimation
+  of item parameters: Application of an EM algorithm. *Psychometrika, 46*(4),
+  443–459. <https://doi.org/10.1007/BF02293801>
+- Cai, L. (2010). High-dimensional exploratory item factor analysis by a
+  Metropolis–Hastings Robbins–Monro algorithm. *Psychometrika, 75*(1), 33–57.
+  <https://doi.org/10.1007/s11336-009-9136-x>
+- Chalmers, R. P. (2012). mirt: A multidimensional item response theory
+  package for the R environment. *Journal of Statistical Software, 48*(6),
+  1–29. <https://doi.org/10.18637/jss.v048.i06>
+- Kim, S. (2006). A comparative study of IRT fixed parameter calibration
+  methods. *Journal of Educational Measurement, 43*(4), 355–381.
+  <https://doi.org/10.1111/j.1745-3984.2006.00021.x>
+- Kim, S., & Kolen, M. J. (2010). Linking item parameters to a base scale.
+  *Journal of Educational Measurement, 47*(2), 164–181.
+  <https://doi.org/10.1111/j.1745-3984.2010.00106.x>
+- Mislevy, R. J., & Wu, P.-K. (1996). *Missing responses and IRT ability
+  estimation: Omits, choice, time limits, and adaptive testing* (ETS Research
+  Report No. RR-96-30-ONR). Educational Testing Service.
+  <https://doi.org/10.1002/j.2333-8504.1996.tb01708.x>
+
+## How each test file is grounded
+
+- `test-kim2006-fixed-anchor-invariant.R` — Fixed-item-parameter linking
+  invariant: anchors keep their old-form values and stay fixed while
+  non-anchors are estimated onto the base scale (Kim, 2006; Bock & Aitkin,
+  1981; Chalmers, 2012).
+- `test-se-hessian-vcov-preservation.R` — `SE = TRUE` observed-information /
+  covariance matrix and the second-order optimality test are preserved through
+  linking on the old, new, and linked models (Bock & Aitkin, 1981; Cai, 2010;
+  Chalmers, 2012).
+- `test-concurrent-missing-robustness.R` — Planned-missing / non-overlapping
+  booklet designs calibrate without error under ignorable missingness, and
+  anchors stay fixed (Bock & Aitkin, 1981; Mislevy & Wu, 1996; Kim, 2006).
+- `test-degenerate-response-robustness.R` — Zero-score, perfect-score, and
+  skewed (near-degenerate) response patterns produce finite item parameters,
+  MAP abilities, and expected scores rather than crashing (Bock & Aitkin,
+  1981; Mislevy & Wu, 1996; Kim, 2006).
+- `test-fixed-parameter-calibration.R` (pre-existing) — End-to-end linking
+  contract fixture (Kim, 2006).

tests/testthat/test-concurrent-missing-robustness.R

@@ -0,0 +1,125 @@
+# Directive item 3 — Concurrent calibration missing-value robustness.
+#
+# In common-item nonequivalent-groups (NEAT) and planned-missing booklet
+# designs, examinees respond to only a subset of the items; the unadministered
+# cells are structurally missing (NA). Under the standard ignorable / MAR
+# assumption for such designs, marginal maximum likelihood integrates the
+# person likelihood over only the OBSERVED responses, so missing-by-design
+# cells neither bias nor break estimation (Bock & Aitkin, 1981; Mislevy & Wu,
+# 1996). This test builds two non-overlapping booklets per form — every
+# examinee sees all anchor items but only one form-specific block of unique
+# items — and asserts that autoFIPC calibrates end-to-end without error while
+# the Kim (2006) fixed-anchor invariant still holds.
+#
+# References (APA 7th):
+#   Bock, R. D., & Aitkin, M. (1981). Marginal maximum likelihood estimation of
+#     item parameters: Application of an EM algorithm. Psychometrika, 46(4),
+#     443-459. https://doi.org/10.1007/BF02293801
+#   Kim, S. (2006). A comparative study of IRT fixed parameter calibration
+#     methods. Journal of Educational Measurement, 43(4), 355-381.
+#     https://doi.org/10.1111/j.1745-3984.2006.00021.x
+#   Mislevy, R. J., & Wu, P.-K. (1996). Missing responses and IRT ability
+#     estimation: Omits, choice, time limits, and adaptive testing (ETS
+#     Research Report No. RR-96-30-ONR). Educational Testing Service.
+#     https://doi.org/10.1002/j.2333-8504.1996.tb01708.x
+
+test_that("autoFIPC calibrates planned-missing booklets and keeps anchors fixed", {
+  skip_on_cran()
+  skip_if_not_installed("mirt")
+
+  set.seed(20260731)
+
+  old_item_names <- paste0("old_", 1:7)
+  new_item_names <- paste0("new_", 1:7)
+  # Items 1:4 are the fully-observed anchors; 5:7 are the form-specific block
+  # split across two booklets.
+  old_common_items <- old_item_names[1:4]
+  new_common_items <- new_item_names[1:4]
+
+  common_a <- c(0.95, 1.18, 0.82, 1.27)
+  common_d <- c(-0.70, -0.15, 0.40, 0.85)
+  old_a <- matrix(c(common_a, 1.05, 0.88, 1.16), ncol = 1)
+  old_d <- c(common_d, -0.35, 0.30, 0.60)
+  new_a <- matrix(c(common_a, 0.90, 1.22, 0.79), ncol = 1)
+  new_d <- c(common_d, 0.20, -0.50, 0.45)
+
+  n_persons <- 1600
+  old_data <- as.data.frame(mirt::simdata(
+    a = old_a, d = old_d,
+    itemtype = rep("2PL", length(old_item_names)), N = n_persons
+  ))
+  new_data <- as.data.frame(mirt::simdata(
+    a = new_a, d = new_d,
+    itemtype = rep("2PL", length(new_item_names)), N = n_persons
+  ))
+  names(old_data) <- old_item_names
+  names(new_data) <- new_item_names
+
+  # Planned-missing booklet spiral: anchors seen by everyone; the two unique
+  # blocks are administered to disjoint halves, the rest are NA (missing by
+  # design, not by response behavior).
+  booklet <- rep(c(1L, 2L), length.out = n_persons)
+  old_block_a <- old_item_names[5:6]
+  old_block_b <- old_item_names[7]
+  new_block_a <- new_item_names[5:6]
+  new_block_b <- new_item_names[7]
+  old_data[booklet == 1L, old_block_b] <- NA
+  old_data[booklet == 2L, old_block_a] <- NA
+  new_data[booklet == 1L, new_block_b] <- NA
+  new_data[booklet == 2L, new_block_a] <- NA
+
+  # Structural missingness must be present but the anchors must stay complete.
+  expect_true(anyNA(old_data))
+  expect_true(anyNA(new_data))
+  expect_false(anyNA(old_data[, old_common_items]))
+  expect_false(anyNA(new_data[, new_common_items]))
+
+  old_model <- mirt::mirt(
+    old_data, 1, itemtype = "2PL", method = "EM", SE = TRUE,
+    verbose = FALSE, technical = list(NCYCLES = 800)
+  )
+  new_model <- mirt::mirt(
+    new_data, 1, itemtype = "2PL", method = "EM", SE = TRUE,
+    verbose = FALSE, technical = list(NCYCLES = 800)
+  )
+
+  # A thrown error here fails the test: the calibration must complete despite
+  # the structurally missing (planned-missing) cells.
+  linked <- aFIPC::autoFIPC(
+    newformXData = new_model,
+    oldformYData = old_model,
+    newformCommonItemNames = new_common_items,
+    oldformCommonItemNames = old_common_items,
+    itemtype = "2PL",
+    checkIPD = FALSE,
+    tryEM = TRUE,
+    freeMEAN = FALSE,
+    forceNormalZeroOne = TRUE,
+    confirmCommonItems = TRUE
+  )
+
+  expect_type(linked, "list")
+  expect_true(all(c("LinkedModel", "ThetaLinkedform") %in% names(linked)))
+
+  # Anchors remain fixed to their old-form values despite the missing blocks.
+  old_values <- mirt::mod2values(old_model)
+  linked_values <- mirt::mod2values(linked$LinkedModel)
+  for (i in seq_along(old_common_items)) {
+    old_fixed <- old_values[
+      old_values$item == old_common_items[i] &
+        old_values$name %in% c("a1", "d"),
+      c("name", "value")
+    ]
+    linked_fixed <- linked_values[
+      linked_values$item == new_common_items[i] &
+        linked_values$name %in% c("a1", "d"),
+      c("name", "value", "est")
+    ]
+    expect_equal(linked_fixed$value, old_fixed$value, tolerance = 1e-6)
+    expect_false(any(linked_fixed$est))
+  }
+
+  # Every calibrated person receives a finite ability estimate on the linked
+  # scale even though each answered only a booklet-sized subset.
+  expect_true(all(is.finite(linked$ThetaLinkedform)))
+})

tests/testthat/test-degenerate-response-robustness.R

@@ -0,0 +1,117 @@
+# Directive item 4 — Zero-score / perfect-score robustness.
+#
+# Real calibration samples contain examinees who answer every item incorrectly
+# (all-0) or every item correctly (all-1), and items whose responses are highly
+# skewed toward one category. Maximum-likelihood ability estimates diverge to
+# +/- infinity for perfect and zero scores, but the marginal / Bayesian
+# machinery aFIPC relies on keeps them finite: MML estimation integrates over a
+# proper population prior (Bock & Aitkin, 1981), and the MAP ability scores
+# autoFIPC returns are shrunk toward that prior and therefore finite even at the
+# boundaries (Mislevy & Wu, 1996). This test injects all-0 and all-1 response
+# vectors plus deliberately extreme (near-degenerate) unique items and asserts
+# that fixed-item calibration (Kim, 2006) neither crashes nor emits non-finite
+# item parameters, ability estimates, or expected scores.
+#
+# Note on scope: a fully CONSTANT item (a single observed category) is not
+# identifiable under any IRT estimator, so this fixture keeps every item
+# non-constant by construction (the injected all-0 and all-1 rows guarantee at
+# least one response in each category) while still exercising the skewed,
+# boundary-heavy regime the directive targets.
+#
+# References (APA 7th):
+#   Bock, R. D., & Aitkin, M. (1981). Marginal maximum likelihood estimation of
+#     item parameters: Application of an EM algorithm. Psychometrika, 46(4),
+#     443-459. https://doi.org/10.1007/BF02293801
+#   Kim, S. (2006). A comparative study of IRT fixed parameter calibration
+#     methods. Journal of Educational Measurement, 43(4), 355-381.
+#     https://doi.org/10.1111/j.1745-3984.2006.00021.x
+#   Mislevy, R. J., & Wu, P.-K. (1996). Missing responses and IRT ability
+#     estimation: Omits, choice, time limits, and adaptive testing (ETS
+#     Research Report No. RR-96-30-ONR). Educational Testing Service.
+#     https://doi.org/10.1002/j.2333-8504.1996.tb01708.x
+
+test_that("autoFIPC survives zero-score, perfect-score, and skewed items", {
+  skip_on_cran()
+  skip_if_not_installed("mirt")
+
+  set.seed(20260801)
+
+  old_item_names <- paste0("old_", 1:6)
+  new_item_names <- paste0("new_", 1:6)
+  old_common_items <- old_item_names[1:4]
+  new_common_items <- new_item_names[1:4]
+
+  common_a <- c(1.00, 0.90, 1.20, 0.85)
+  common_d <- c(-0.60, -0.10, 0.30, 0.65)
+  # Unique items 5:6 are deliberately extreme in difficulty to create heavily
+  # skewed, near-degenerate response columns.
+  old_a <- matrix(c(common_a, 1.15, 0.80), ncol = 1)
+  old_d <- c(common_d, 3.2, -3.0)
+  new_a <- matrix(c(common_a, 0.95, 1.10), ncol = 1)
+  new_d <- c(common_d, -3.1, 3.3)
+
+  old_data <- as.data.frame(mirt::simdata(
+    a = old_a, d = old_d,
+    itemtype = rep("2PL", length(old_item_names)), N = 1500
+  ))
+  new_data <- as.data.frame(mirt::simdata(
+    a = new_a, d = new_d,
+    itemtype = rep("2PL", length(new_item_names)), N = 1500
+  ))
+  names(old_data) <- old_item_names
+  names(new_data) <- new_item_names
+
+  # Inject boundary examinees: rows 1:3 all-incorrect, rows 4:6 all-correct.
+  # This also guarantees every item has >=3 zeros and >=3 ones (non-constant).
+  old_data[1:3, ] <- 0
+  old_data[4:6, ] <- 1
+  new_data[1:3, ] <- 0
+  new_data[4:6, ] <- 1
+
+  # Sanity guard: no item collapsed to a single category.
+  expect_true(all(vapply(old_data, function(x) length(unique(x)) >= 2, logical(1))))
+  expect_true(all(vapply(new_data, function(x) length(unique(x)) >= 2, logical(1))))
+
+  old_model <- mirt::mirt(
+    old_data, 1, itemtype = "2PL", method = "EM", SE = TRUE,
+    verbose = FALSE, technical = list(NCYCLES = 800)
+  )
+  new_model <- mirt::mirt(
+    new_data, 1, itemtype = "2PL", method = "EM", SE = TRUE,
+    verbose = FALSE, technical = list(NCYCLES = 800)
+  )
+
+  # A thrown error here fails the test: calibration must complete despite the
+  # boundary (all-0 / all-1) examinees and skewed items.
+  linked <- aFIPC::autoFIPC(
+    newformXData = new_model,
+    oldformYData = old_model,
+    newformCommonItemNames = new_common_items,
+    oldformCommonItemNames = old_common_items,
+    itemtype = "2PL",
+    checkIPD = FALSE,
+    tryEM = TRUE,
+    freeMEAN = FALSE,
+    forceNormalZeroOne = TRUE,
+    confirmCommonItems = TRUE
+  )
+
+  expect_type(linked, "list")
+
+  # Linked item parameters must all be finite despite the boundary examinees.
+  linked_values <- mirt::mod2values(linked$LinkedModel)
+  item_params <- linked_values[
+    linked_values$name %in% c("a1", "d"), "value"
+  ]
+  expect_true(all(is.finite(item_params)))
+
+  # MAP ability estimates on all three forms remain finite at the boundaries.
+  expect_true(all(is.finite(linked$ThetaOldform)))
+  expect_true(all(is.finite(linked$ThetaNewform)))
+  expect_true(all(is.finite(linked$ThetaLinkedform)))
+
+  # Expected test scores remain finite as well.
+  expect_true(all(is.finite(linked$ExpectedScoreOldform)))
+  expect_true(all(is.finite(linked$ExpectedScoreNewform)))
+  expect_true(all(is.finite(linked$ExpectedScoreLinkedform)))
+})