@article {2739, title = {The Optimal Item Pool Design in Multistage Computerized Adaptive Tests With the p-Optimality Method}, journal = {Educational and Psychological Measurement}, volume = {80}, number = {5}, year = {2020}, pages = {955-974}, abstract = {The present study extended the p-optimality method to the multistage computerized adaptive test (MST) context in developing optimal item pools to support different MST panel designs under different test configurations. Using the Rasch model, simulated optimal item pools were generated with and without practical constraints of exposure control. A total number of 72 simulated optimal item pools were generated and evaluated by an overall sample and conditional sample using various statistical measures. Results showed that the optimal item pools built with the p-optimality method provide sufficient measurement accuracy under all simulated MST panel designs. Exposure control affected the item pool size, but not the item distributions and item pool characteristics. This study demonstrated that the p-optimality method can adapt to MST item pool design, facilitate the MST assembly process, and improve its scoring accuracy.}, doi = {10.1177/0013164419901292}, url = {https://doi.org/10.1177/0013164419901292}, author = {Lihong Yang and Mark D. Reckase} } @article {2733, title = {Stratified Item Selection Methods in Cognitive Diagnosis Computerized Adaptive Testing}, journal = {Applied Psychological Measurement}, volume = {44}, number = {5}, year = {2020}, pages = {346-361}, abstract = {Cognitive diagnostic computerized adaptive testing (CD-CAT) aims to obtain more useful diagnostic information by taking advantages of computerized adaptive testing (CAT). Cognitive diagnosis models (CDMs) have been developed to classify examinees into the correct proficiency classes so as to get more efficient remediation, whereas CAT tailors optimal items to the examinee{\textquoteright}s mastery profile. The item selection method is the key factor of the CD-CAT procedure. In recent years, a large number of parametric/nonparametric item selection methods have been proposed. In this article, the authors proposed a series of stratified item selection methods in CD-CAT, which are combined with posterior-weighted Kullback{\textendash}Leibler (PWKL), nonparametric item selection (NPS), and weighted nonparametric item selection (WNPS) methods, and named S-PWKL, S-NPS, and S-WNPS, respectively. Two different types of stratification indices were used: original versus novel. The performances of the proposed item selection methods were evaluated via simulation studies and compared with the PWKL, NPS, and WNPS methods without stratification. Manipulated conditions included calibration sample size, item quality, number of attributes, number of strata, and data generation models. Results indicated that the S-WNPS and S-NPS methods performed similarly, and both outperformed the S-PWKL method. And item selection methods with novel stratification indices performed slightly better than the ones with original stratification indices, and those without stratification performed the worst.}, doi = {10.1177/0146621619893783}, url = {https://doi.org/10.1177/0146621619893783}, author = {Jing Yang and Hua-Hua Chang and Jian Tao and Ningzhong Shi} } @article {2723, title = {Computerized Adaptive Testing for Cognitively Based Multiple-Choice Data}, journal = {Applied Psychological Measurement}, volume = {43}, number = {5}, year = {2019}, pages = {388-401}, abstract = {Cognitive diagnosis models (CDMs) are latent class models that hold great promise for providing diagnostic information about student knowledge profiles. The increasing use of computers in classrooms enhances the advantages of CDMs for more efficient diagnostic testing by using adaptive algorithms, referred to as cognitive diagnosis computerized adaptive testing (CD-CAT). When multiple-choice items are involved, CD-CAT can be further improved by using polytomous scoring (i.e., considering the specific options students choose), instead of dichotomous scoring (i.e., marking answers as either right or wrong). In this study, the authors propose and evaluate the performance of the Jensen{\textendash}Shannon divergence (JSD) index as an item selection method for the multiple-choice deterministic inputs, noisy {\textquotedblleft}and{\textquotedblright} gate (MC-DINA) model. Attribute classification accuracy and item usage are evaluated under different conditions of item quality and test termination rule. The proposed approach is compared with the random selection method and an approximate approach based on dichotomized responses. The results show that under the MC-DINA model, JSD improves the attribute classification accuracy significantly by considering the information from distractors, even with a very short test length. This result has important implications in practical classroom settings as it can allow for dramatically reduced testing times, thus resulting in more targeted learning opportunities.}, doi = {10.1177/0146621618798665}, url = {https://doi.org/10.1177/0146621618798665}, author = {Hulya D. Yigit and Miguel A. Sorrel and Jimmy de la Torre} } @article {2624, title = {Latent Class Analysis of Recurrent Events in Problem-Solving Items}, journal = {Applied Psychological Measurement}, volume = {42}, number = {6}, year = {2018}, pages = {478-498}, abstract = {Computer-based assessment of complex problem-solving abilities is becoming more and more popular. In such an assessment, the entire problem-solving process of an examinee is recorded, providing detailed information about the individual, such as behavioral patterns, speed, and learning trajectory. The problem-solving processes are recorded in a computer log file which is a time-stamped documentation of events related to task completion. As opposed to cross-sectional response data from traditional tests, process data in log files are massive and irregularly structured, calling for effective exploratory data analysis methods. Motivated by a specific complex problem-solving item {\textquotedblleft}Climate Control{\textquotedblright} in the 2012 Programme for International Student Assessment, the authors propose a latent class analysis approach to analyzing the events occurred in the problem-solving processes. The exploratory latent class analysis yields meaningful latent classes. Simulation studies are conducted to evaluate the proposed approach.}, doi = {10.1177/0146621617748325}, url = {https://doi.org/10.1177/0146621617748325}, author = {Haochen Xu and Guanhua Fang and Yunxiao Chen and Jingchen Liu and Zhiliang Ying} } @conference {2656, title = {Efficiency of Item Selection in CD-CAT Based on Conjunctive Bayesian Network Modeling Hierarchical attributes}, booktitle = {IACAT 2017 Conference}, year = {2017}, month = {08/2017}, publisher = {Niigata Seiryo University}, organization = {Niigata Seiryo University}, address = {Niigata, Japan}, abstract = {

Cognitive diagnosis models (CDM) aim to diagnosis examinee\’s mastery status of multiple fine-grained skills. As new development for cognitive diagnosis methods emerges, much attention is given to cognitive diagnostic computerized adaptive testing (CD-CAT) as well. The topics such as item selection methods, item exposure control strategies, and online calibration methods, which have been wellstudied for traditional item response theory (IRT) based CAT, are also investigated in the context of CD-CAT (e.g., Xu, Chang, \& Douglas, 2003; Wang, Chang, \& Huebner, 2011; Chen et al., 2012).

In CDM framework, some researchers suggest to model structural relationship between cognitive skills, or namely, attributes. Especially, attributes can be hierarchical, such that some attributes must be acquired before the subsequent ones are mastered. For example, in mathematics, addition must be mastered before multiplication, which gives a hierarchy model for addition skill and multiplication skill. Recently, new CDMs considering attribute hierarchies have been suggested including the Attribute Hierarchy Method (AHM; Leighton, Gierl, \& Hunka, 2004) and the Hierarchical Diagnostic Classification Models (HDCM; Templin \& Bradshaw, 2014).

Bayesian Networks (BN), the probabilistic graphical models representing the relationship of a set of random variables using a directed acyclic graph with conditional probability distributions, also provide an efficient framework for modeling the relationship between attributes (Culbertson, 2016). Among various BNs, conjunctive Bayesian network (CBN; Beerenwinkel, Eriksson, \& Sturmfels, 2007) is a special kind of BN, which assumes partial ordering between occurrences of events and conjunctive constraints between them.

In this study, we propose using CBN for modeling attribute hierarchies and discuss the advantage of CBN for CDM. We then explore the impact of the CBN modeling on the efficiency of item selection methods for CD-CAT when the attributes are truly hierarchical. To this end, two simulation studies, one for fixed-length CAT and another for variable-length CAT, are conducted. For each studies, two attribute hierarchy structures with 5 and 8 attributes are assumed. Among the various item selection methods developed for CD-CAT, six algorithms are considered: posterior-weighted Kullback-Leibler index (PWKL; Cheng, 2009), the modified PWKL index (MPWKL; Kaplan, de la Torre, Barrada, 2015), Shannon entropy (SHE; Tatsuoka, 2002), mutual information (MI; Wang, 2013), posterior-weighted CDM discrimination index (PWCDI; Zheng \& Chang, 2016) and posterior-weighted attribute-level CDM discrimination index (PWACDI; Zheng \& Chang, 2016). The impact of Q-matrix structure, item quality, and test termination rules on the efficiency of item selection algorithms is also investigated. Evaluation measures include the attribute classification accuracy (fixed-length experiment) and test length of CDCAT until stopping (variable-length experiment).

The results of the study indicate that the efficiency of item selection is improved by directly modeling the attribute hierarchies using CBN. The test length until achieving diagnosis probability threshold was reduced to 50-70\% for CBN based CAT compared to the CD-CAT assuming independence of attributes. The magnitude of improvement is greater when the cognitive model of the test includes more attributes and when the test length is shorter. We conclude by discussing how Q-matrix structure, item quality, and test termination rules affect the efficiency.

References

Beerenwinkel, N., Eriksson, N., \& Sturmfels, B. (2007). Conjunctive bayesian networks. Bernoulli, 893- 909.

Chen, P., Xin, T., Wang, C., \& Chang, H. H. (2012). Online calibration methods for the DINA model with independent attributes in CD-CAT. Psychometrika, 77(2), 201-222.

Cheng, Y. (2009). When cognitive diagnosis meets computerized adaptive testing: CD-CAT. Psychometrika, 74(4), 619-632.

Culbertson, M. J. (2016). Bayesian networks in educational assessment: the state of the field. Applied Psychological Measurement, 40(1), 3-21.

Kaplan, M., de la Torre, J., \& Barrada, J. R. (2015). New item selection methods for cognitive diagnosis computerized adaptive testing. Applied Psychological Measurement, 39(3), 167-188.

Leighton, J. P., Gierl, M. J., \& Hunka, S. M. (2004). The attribute hierarchy method for cognitive assessment: a variation on Tatsuoka\&$\#$39;s rule-space approach. Journal of Educational Measurement, 41(3), 205-237.

Tatsuoka, C. (2002). Data analytic methods for latent partially ordered classification models. Journal of the Royal Statistical Society: Series C (Applied Statistics), 51(3), 337-350.

Templin, J., \& Bradshaw, L. (2014). Hierarchical diagnostic classification models: A family of models for estimating and testing attribute hierarchies. Psychometrika, 79(2), 317-339. Wang, C. (2013). Mutual information item selection method in cognitive diagnostic computerized adaptive testing with short test length. Educational and Psychological Measurement, 73(6), 1017-1035.

Wang, C., Chang, H. H., \& Huebner, A. (2011). Restrictive stochastic item selection methods in cognitive diagnostic computerized adaptive testing. Journal of Educational Measurement, 48(3), 255-273.

Xu, X., Chang, H., \& Douglas, J. (2003, April). A simulation study to compare CAT strategies for cognitive diagnosis. Paper presented at the annual meeting of National Council on Measurement in Education, Chicago.

Zheng, C., \& Chang, H. H. (2016). High-efficiency response distribution\–based item selection algorithms for short-length cognitive diagnostic computerized adaptive testing. Applied Psychological Measurement, 40(8), 608-624.

Session Video

}, keywords = {CD-CAT, Conjuctive Bayesian Network Modeling, item selection}, url = {https://drive.google.com/open?id=1RbO2gd4aULqsSgRi_VZudNN_edX82NeD}, author = {Soo-Yun Han and Yun Joo Yoo} } @conference {2660, title = {MHK-MST Design and the Related Simulation Study}, booktitle = {IACAT 2017 Conference}, year = {2017}, month = {08/2017}, publisher = {Niigata Seiryo University}, organization = {Niigata Seiryo University}, address = {Niigata, Japan}, abstract = {

The MHK is a national standardized exam that tests and rates Chinese language proficiency. It assesses non-native Chinese minorities\’ abilities in using the Chinese language in their daily, academic and professional lives; Computerized multistage adaptive testing (MST) is a combination of conventional paper-and-pencil (P\&P) and item level computerized adaptive test (CAT), it is a kind of test forms based on computerized technology, take the item set as the scoring unit. It can be said that, MST estimate the Ability extreme value more accurate than conventional paper-and-pencil (P\&P), also used the CAT auto-adapted characteristic to reduce the examination length and the score time of report. At present, MST has used in some large test, like Uniform CPA Examination and Graduate Record Examination(GRE). Therefore, it is necessary to develop the MST of application in China.

Based on consideration of the MHK characteristics and its future development, the researchers start with design of MHK-MST. This simulation study is conducted to validate the performance of the MHK -MST system. Real difficulty parameters of MHK items and the simulated ability parameters of the candidates are used to generate the original score matrix and the item modules are delivered to the candidates following the adaptive procedures set according to the path rules. This simulation study provides a sound basis for the implementation of MHK-MST.

Session Video

}, keywords = {language testing, MHK, multistage testing}, author = {Ling Yuyu and Zhou Chenglin and Ren Jie} } @conference {2639, title = {A Simulation Study to Compare Classification Method in Cognitive Diagnosis Computerized Adaptive Testing}, booktitle = {IACAT 2017 Conference}, year = {2017}, month = {08/2017}, publisher = {Niigata Seiryo University}, organization = {Niigata Seiryo University}, address = {Niigata, Japan}, abstract = {

Cognitive Diagnostic Computerized Adaptive Testing (CD-CAT) combines the strengths of both CAT and cognitive diagnosis. Cognitive diagnosis models that can be viewed as restricted latent class models have been developed to classify the examinees into the correct profile of skills that have been mastered and those that have not so as to get more efficient remediation. Chiu \& Douglas (2013) introduces a nonparametric procedure that only requires specification of Q-matrix to classify by proximity to ideal response pattern. In this article, we compare nonparametric procedure with common profile estimation method like maximum a posterior (MAP) in CD-CAT. Simulation studies consider a variety of Q-matrix structure, the number of attributes, ways to generate attribute profiles,\ and item quality. Results indicate that nonparametric procedure consistently gets the higher pattern and attribute recovery rate in nearly all conditions.

References

Chiu, C.-Y., \& Douglas, J. (2013). A nonparametric approach to cognitive diagnosis by proximity to ideal response patterns. Journal of Classification, 30, 225-250. doi: 10.1007/s00357-013-9132-9

Session Video

}, url = {https://drive.google.com/open?id=1jCL3fPZLgzIdwvEk20D-FliZ15OTUtpr}, author = {Jing Yang and Jian Tao and Hua-Hua Chang and Ning-Zhong Shi} } @article {2383, title = {A Comparison of IRT Proficiency Estimation Methods Under Adaptive Multistage Testing}, journal = {Journal of Educational Measurement}, volume = {52}, number = {1}, year = {2015}, pages = {70{\textendash}79}, abstract = {This inquiry is an investigation of item response theory (IRT) proficiency estimators{\textquoteright} accuracy under multistage testing (MST). We chose a two-stage MST design that includes four modules (one at Stage 1, three at Stage 2) and three difficulty paths (low, middle, high). We assembled various two-stage MST panels (i.e., forms) by manipulating two assembly conditions in each module, such as difficulty level and module length. For each panel, we investigated the accuracy of examinees{\textquoteright} proficiency levels derived from seven IRT proficiency estimators. The choice of Bayesian (prior) versus non-Bayesian (no prior) estimators was of more practical significance than the choice of number-correct versus item-pattern scoring estimators. The Bayesian estimators were slightly more efficient than the non-Bayesian estimators, resulting in smaller overall error. Possible score changes caused by the use of different proficiency estimators would be nonnegligible, particularly for low- and high-performing examinees.}, issn = {1745-3984}, doi = {10.1111/jedm.12063}, url = {http://dx.doi.org/10.1111/jedm.12063}, author = {Kim, Sooyeon and Moses, Tim and Yoo, Hanwook (Henry)} } @article {2348, title = {Online Item Calibration for Q-Matrix in CD-CAT}, journal = {Applied Psychological Measurement}, volume = {39}, number = {1}, year = {2015}, pages = {5-15}, abstract = {

Item replenishment is important for maintaining a large-scale item bank. In this article, the authors consider calibrating new items based on pre-calibrated operational items under the deterministic inputs, noisy-and-gate model, the specification of which includes the so-called -matrix, as well as the slipping and guessing parameters. Making use of the maximum likelihood and Bayesian estimators for the latent knowledge states, the authors propose two methods for the calibration. These methods are applicable to both traditional paper\–pencil\–based tests, for which the selection of operational items is prefixed, and computerized adaptive tests, for which the selection of operational items is sequential and random. Extensive simulations are done to assess and to compare the performance of these approaches. Extensions to other diagnostic classification models are also discussed.

}, doi = {10.1177/0146621613513065}, url = {http://apm.sagepub.com/content/39/1/5.abstract}, author = {Chen, Yunxiao and Liu, Jingchen and Ying, Zhiliang} } @book {2480, title = {Computerized multistage testing: Theory and applications}, year = {2014}, publisher = {CRC Press}, organization = {CRC Press}, address = {Boca Raton FL}, isbn = {13-978-1-4665-0577-3}, author = {Duanli Yan and Alina A von Davier and Charles Lewis} } @article {2324, title = {Multidimensional CAT Item Selection Methods for Domain Scores and Composite Scores With Item Exposure Control and Content Constraints}, journal = {Journal of Educational Measurement}, volume = {51}, number = {1}, year = {2014}, pages = {18{\textendash}38}, abstract = {

The intent of this research was to find an item selection procedure in the multidimensional computer adaptive testing (CAT) framework that yielded higher precision for both the domain and composite abilities, had a higher usage of the item pool, and controlled the exposure rate. Five multidimensional CAT item selection procedures (minimum angle; volume; minimum error variance of the linear combination; minimum error variance of the composite score with optimized weight; and Kullback-Leibler information) were studied and compared with two methods for item exposure control (the Sympson-Hetter procedure and the fixed-rate procedure, the latter simply refers to putting a limit on the item exposure rate) using simulated data. The maximum priority index method was used for the content constraints. Results showed that the Sympson-Hetter procedure yielded better precision than the fixed-rate procedure but had much lower item pool usage and took more time. The five item selection procedures performed similarly under Sympson-Hetter. For the fixed-rate procedure, there was a trade-off between the precision of the ability estimates and the item pool usage: the five procedures had different patterns. It was found that (1) Kullback-Leibler had better precision but lower item pool usage; (2) minimum angle and volume had balanced precision and item pool usage; and (3) the two methods minimizing the error variance had the best item pool usage and comparable overall score recovery but less precision for certain domains. The priority index for content constraints and item exposure was implemented successfully.

}, issn = {1745-3984}, doi = {10.1111/jedm.12032}, url = {http://dx.doi.org/10.1111/jedm.12032}, author = {Yao, Lihua} } @article {2344, title = {Using Multidimensional CAT to Administer a Short, Yet Precise, Screening Test}, journal = {Applied Psychological Measurement}, volume = {38}, number = {8}, year = {2014}, pages = {614-631}, abstract = {

Multidimensional computerized adaptive testing (MCAT) provides a mechanism by which the simultaneous goals of accurate prediction and minimal testing time for a screening test could both be met. This article demonstrates the use of MCAT to administer a screening test for the Computerized Adaptive Testing\–Armed Services Vocational Aptitude Battery (CAT-ASVAB) under a variety of manipulated conditions. CAT-ASVAB is a test battery administered via unidimensional CAT (UCAT) that is used to qualify applicants for entry into the U.S. military and assign them to jobs. The primary research question being evaluated is whether the use of MCAT to administer a screening test can lead to significant reductions in testing time from the full-length selection test, without significant losses in score precision. Different stopping rules, item selection methods, content constraints, time constraints, and population distributions for the MCAT administration are evaluated through simulation, and compared with results from a regular full-length UCAT administration.

}, doi = {10.1177/0146621614541514}, url = {http://apm.sagepub.com/content/38/8/614.abstract}, author = {Yao, Lihua and Pommerich, Mary and Segall, Daniel O.} } @article {2265, title = {Comparing the Performance of Five Multidimensional CAT Selection Procedures With Different Stopping Rules}, journal = {Applied Psychological Measurement}, volume = {37}, number = {1}, year = {2013}, pages = {3-23}, abstract = {

Through simulated data, five multidimensional computerized adaptive testing (MCAT) selection procedures with varying test lengths are examined and compared using different stopping rules. Fixed item exposure rates are used for all the items, and the Priority Index (PI) method is used for the content constraints. Two stopping rules, standard error (SE) and predicted standard error reduction (PSER), are proposed; each MCAT selection process is stopped if either the required precision has been achieved or the selected number of items has reached the maximum limit. The five procedures are as follows: minimum angle (Ag), volume (Vm), minimize the error variance of the linear combination (V 1), minimize the error variance of the composite score with the optimized weight (V 2), and Kullback\–Leibler (KL) information. The recovery for the domain scores or content scores and their overall score, test length, and test reliability are compared across the five MCAT procedures and between the two stopping rules. It is found that the two stopping rules are implemented successfully and that KL uses the least number of items to reach the same precision level, followed by Vm; Ag uses the largest number of items. On average, to reach a precision of SE = .35, 40, 55, 63, 63, and 82 items are needed for KL, Vm, V 1, V 2, and Ag, respectively, for the SE stopping rule. PSER yields 38, 45, 53, 58, and 68 items for KL, Vm, V 1, V 2, and Ag, respectively; PSER yields only slightly worse results than SE, but with much fewer items. Overall, KL is recommended for varying-length MCAT.

}, doi = {10.1177/0146621612455687}, url = {http://apm.sagepub.com/content/37/1/3.abstract}, author = {Yao, Lihua} } @article {2266, title = {The Influence of Item Calibration Error on Variable-Length Computerized Adaptive Testing}, journal = {Applied Psychological Measurement}, volume = {37}, number = {1}, year = {2013}, pages = {24-40}, abstract = {

Variable-length computerized adaptive testing (VL-CAT) allows both items and test length to be \“tailored\” to examinees, thereby achieving the measurement goal (e.g., scoring precision or classification) with as few items as possible. Several popular test termination rules depend on the standard error of the ability estimate, which in turn depends on the item parameter values. However, items are chosen on the basis of their parameter estimates, and capitalization on chance may occur. In this article, the authors investigated the effects of capitalization on chance on test length and classification accuracy in several VL-CAT simulations. The results confirm that capitalization on chance occurs in VL-CAT and has complex effects on test length, ability estimation, and classification accuracy. These results have important implications for the design and implementation of VL-CATs.

}, doi = {10.1177/0146621612461727}, url = {http://apm.sagepub.com/content/37/1/24.abstract}, author = {Patton, Jeffrey M. and Ying Cheng, and Yuan, Ke-Hai and Diao, Qi} } @article {2166, title = {An Empirical Evaluation of the Slip Correction in the Four Parameter Logistic Models With Computerized Adaptive Testing}, journal = {Applied Psychological Measurement}, volume = {36}, number = {2}, year = {2012}, pages = {75-87}, abstract = {

In a selected response test, aberrant responses such as careless errors and lucky guesses might cause error in ability estimation because these responses do not actually reflect the knowledge that examinees possess. In a computerized adaptive test (CAT), these aberrant responses could further cause serious estimation error due to dynamic item administration. To enhance the robust performance of CAT against aberrant responses, Barton and Lord proposed the four-parameter logistic (4PL) item response theory (IRT) model. However, most studies relevant to the 4PL IRT model were conducted based on simulation experiments. This study attempts to investigate the performance of the 4PL IRT model as a slip-correction mechanism with an empirical experiment. The results showed that the 4PL IRT model could not only reduce the problematic underestimation of the examinees\’ ability introduced by careless mistakes in practical situations but also improve measurement efficiency.

}, doi = {10.1177/0146621611432862}, url = {http://apm.sagepub.com/content/36/2/75.abstract}, author = {Yen, Yung-Chin and Ho, Rong-Guey and Laio, Wen-Wei and Chen, Li-Ju and Kuo, Ching-Chin} } @booklet {154, title = {Cross-cultural development of an item list for computer-adaptive testing of fatigue in oncological patients}, journal = {Health and Quality of Life Outcomes}, volume = {9}, number = {1}, year = {2011}, note = {Health Qual Life Outcomes. 2011 Mar 29;9(1):19.}, month = {March 29, 2011}, pages = {10}, edition = {2011/03/31}, abstract = {ABSTRACT: INTRODUCTION: Within an ongoing project of the EORTC Quality of Life Group, we are developing computerized adaptive test (CAT) measures for the QLQ-C30 scales. These new CAT measures are conceptualised to reflect the same constructs as the QLQ-C30 scales. Accordingly, the Fatigue-CAT is intended to capture physical and general fatigue. METHODS: The EORTC approach to CAT development comprises four phases (literature search, operationalisation, pre-testing, and field testing). Phases I-III are described in detail in this paper. A literature search for fatigue items was performed in major medical databases. After refinement through several expert panels, the remaining items were used as the basis for adapting items and/or formulating new items fitting the EORTC item style. To obtain feedback from patients with cancer, these English items were translated into Danish, French, German, and Spanish and tested in the respective countries. RESULTS: Based on the literature search a list containing 588 items was generated. After a comprehensive item selection procedure focusing on content, redundancy, item clarity and item difficulty a list of 44 fatigue items was generated. Patient interviews (n=52) resulted in 12 revisions of wording and translations. DISCUSSION: The item list developed in phases I-III will be further investigated within a field-testing phase (IV) to examine psychometric characteristics and to fit an item response theory model. The Fatigue CAT based on this item bank will provide scores that are backward-compatible to the original QLQ-C30 fatigue scale.}, isbn = {1477-7525 (Electronic)1477-7525 (Linking)}, author = {Giesinger, J. M. and Petersen, M. A. and Groenvold, M. and Aaronson, N. K. and Arraras, J. I. and Conroy, T. and Gamper, E. M. and Kemmler, G. and King, M. T. and Oberguggenberger, A. S. and Velikova, G. and Young, T. and Holzner, B. and Eortc-Qlg, E. O.} } @article {819, title = {Development and evaluation of a confidence-weighting computerized adaptive testing}, journal = {Educational Technology \& Society}, volume = {13(3)}, year = {2010}, pages = {163{\textendash}176}, author = {Yen, Y. C. and Ho, R. G. and Chen, L. J. and Chou, K. Y. and Chen, Y. L.} } @article {46, title = {Development and validation of patient-reported outcome measures for sleep disturbance and sleep-related impairments}, journal = {Sleep}, volume = {33}, number = {6}, year = {2010}, note = {Buysse, Daniel JYu, LanMoul, Douglas EGermain, AnneStover, AngelaDodds, Nathan EJohnston, Kelly LShablesky-Cade, Melissa APilkonis, Paul AAR052155/AR/NIAMS NIH HHS/United StatesU01AR52155/AR/NIAMS NIH HHS/United StatesU01AR52158/AR/NIAMS NIH HHS/United StatesU01AR52170/AR/NIAMS NIH HHS/United StatesU01AR52171/AR/NIAMS NIH HHS/United StatesU01AR52177/AR/NIAMS NIH HHS/United StatesU01AR52181/AR/NIAMS NIH HHS/United StatesU01AR52186/AR/NIAMS NIH HHS/United StatesResearch Support, N.I.H., ExtramuralValidation StudiesUnited StatesSleepSleep. 2010 Jun 1;33(6):781-92.}, month = {Jun 1}, pages = {781-92}, edition = {2010/06/17}, abstract = {STUDY OBJECTIVES: To develop an archive of self-report questions assessing sleep disturbance and sleep-related impairments (SRI), to develop item banks from this archive, and to validate and calibrate the item banks using classic validation techniques and item response theory analyses in a sample of clinical and community participants. DESIGN: Cross-sectional self-report study. SETTING: Academic medical center and participant homes. PARTICIPANTS: One thousand nine hundred ninety-three adults recruited from an Internet polling sample and 259 adults recruited from medical, psychiatric, and sleep clinics. INTERVENTIONS: None. MEASUREMENTS AND RESULTS: This study was part of PROMIS (Patient-Reported Outcomes Information System), a National Institutes of Health Roadmap initiative. Self-report item banks were developed through an iterative process of literature searches, collecting and sorting items, expert content review, qualitative patient research, and pilot testing. Internal consistency, convergent validity, and exploratory and confirmatory factor analysis were examined in the resulting item banks. Factor analyses identified 2 preliminary item banks, sleep disturbance and SRI. Item response theory analyses and expert content review narrowed the item banks to 27 and 16 items, respectively. Validity of the item banks was supported by moderate to high correlations with existing scales and by significant differences in sleep disturbance and SRI scores between participants with and without sleep disorders. CONCLUSIONS: The PROMIS sleep disturbance and SRI item banks have excellent measurement properties and may prove to be useful for assessing general aspects of sleep and SRI with various groups of patients and interventions.}, keywords = {*Outcome Assessment (Health Care), *Self Disclosure, Adult, Aged, Aged, 80 and over, Cross-Sectional Studies, Factor Analysis, Statistical, Female, Humans, Male, Middle Aged, Psychometrics, Questionnaires, Reproducibility of Results, Sleep Disorders/*diagnosis, Young Adult}, isbn = {0161-8105 (Print)0161-8105 (Linking)}, author = {Buysse, D. J. and Yu, L. and Moul, D. E. and Germain, A. and Stover, A. and Dodds, N. E. and Johnston, K. L. and Shablesky-Cade, M. A. and Pilkonis, P. A.} } @article {312, title = {Development of computerized adaptive testing (CAT) for the EORTC QLQ-C30 physical functioning dimension}, journal = {Quality of Life Research }, volume = {20}, number = {4}, year = {2010}, note = {Qual Life Res. 2010 Oct 23.}, pages = {479-490}, edition = {2010/10/26}, abstract = {PURPOSE: Computerized adaptive test (CAT) methods, based on item response theory (IRT), enable a patient-reported outcome instrument to be adapted to the individual patient while maintaining direct comparability of scores. The EORTC Quality of Life Group is developing a CAT version of the widely used EORTC QLQ-C30. We present the development and psychometric validation of the item pool for the first of the scales, physical functioning (PF). METHODS: Initial developments (including literature search and patient and expert evaluations) resulted in 56 candidate items. Responses to these items were collected from 1,176 patients with cancer from Denmark, France, Germany, Italy, Taiwan, and the United Kingdom. The items were evaluated with regard to psychometric properties. RESULTS: Evaluations showed that 31 of the items could be included in a unidimensional IRT model with acceptable fit and good content coverage, although the pool may lack items at the upper extreme (good PF). There were several findings of significant differential item functioning (DIF). However, the DIF findings appeared to have little impact on the PF estimation. CONCLUSIONS: We have established an item pool for CAT measurement of PF and believe that this CAT instrument will clearly improve the EORTC measurement of PF.}, isbn = {1573-2649 (Electronic)0962-9343 (Linking)}, author = {Petersen, M. A. and Groenvold, M. and Aaronson, N. K. and Chie, W. C. and Conroy, T. and Costantini, A. and Fayers, P. and Helbostad, J. and Holzner, B. and Kaasa, S. and Singer, S. and Velikova, G. and Young, T.} } @article {2187, title = {Improving Cognitive Diagnostic Computerized Adaptive Testing by Balancing Attribute Coverage: The Modified Maximum Global Discrimination Index Method}, journal = {Educational and Psychological Measurement}, volume = {70}, number = {6}, year = {2010}, pages = {902-913}, abstract = {

This article proposes a new item selection method, namely, the modified maximum global discrimination index (MMGDI) method, for cognitive diagnostic computerized adaptive testing (CD-CAT). The new method captures two aspects of the appeal of an item: (a) the amount of contribution it can make toward adequate coverage of every attribute and (b) the amount of contribution it can make toward recovering the latent cognitive profile. A simulation study shows that the new method ensures adequate coverage of every attribute, which improves the validity of the test scores, and defensibility of the proposed uses of the test. Furthermore, compared with the original global discrimination index method, the MMGDI method improves the recovery rate of each attribute and of the entire cognitive profile, especially the latter. Therefore, the new method improves both the validity and reliability of the test scores from a CD-CAT program.

}, doi = {10.1177/0013164410366693}, url = {http://epm.sagepub.com/content/70/6/902.abstract}, author = {Ying Cheng,} } @article {2182, title = {Constraint-Weighted a-Stratification for Computerized Adaptive Testing With Nonstatistical Constraints}, journal = {Educational and Psychological Measurement}, volume = {69}, number = {1}, year = {2009}, pages = {35-49}, abstract = {

a-stratification is a method that utilizes items with small discrimination (a) parameters early in an exam and those with higher a values when more is learned about the ability parameter. It can achieve much better item usage than the maximum information criterion (MIC). To make a-stratification more practical and more widely applicable, a method for weighting the item selection process in a-stratification as a means of satisfying multiple test constraints is proposed. This method is studied in simulation against an analogous method without stratification as well as a-stratification using descending-rather than ascending-a procedures. In addition, a variation of a-stratification that allows for unbalanced usage of a parameters is included in the study to examine the trade-off between efficiency and exposure control. Finally, MIC and randomized item selection are included as baseline measures. Results indicate that the weighting mechanism successfully addresses the constraints, that stratification helps to a great extent balancing exposure rates, and that the ascending-a design improves measurement precision.

}, doi = {10.1177/0013164408322030}, url = {http://epm.sagepub.com/content/69/1/35.abstract}, author = {Ying Cheng, and Chang, Hua-Hua and Douglas, Jeffrey and Fanmin Guo,} } @article {2179, title = {Comparability of Computer-Based and Paper-and-Pencil Testing in K{\textendash}12 Reading Assessments}, journal = {Educational and Psychological Measurement}, volume = {68}, number = {1}, year = {2008}, pages = {5-24}, abstract = {

In recent years, computer-based testing (CBT) has grown in popularity, is increasingly being implemented across the United States, and will likely become the primary mode for delivering tests in the future. Although CBT offers many advantages over traditional paper-and-pencil testing, assessment experts, researchers, practitioners, and users have expressed concern about the comparability of scores between the two test administration modes. To help provide an answer to this issue, a meta-analysis was conducted to synthesize the administration mode effects of CBTs and paper-and-pencil tests on K\—12 student reading assessments. Findings indicate that the administration mode had no statistically significant effect on K\—12 student reading achievement scores. Four moderator variables\—study design, sample size, computer delivery algorithm, and computer practice\—made statistically significant contributions to predicting effect size. Three moderator variables\—grade level, type of test, and computer delivery method\—did not affect the differences in reading scores between test modes.

}, doi = {10.1177/0013164407305592}, url = {http://epm.sagepub.com/content/68/1/5.abstract}, author = {Shudong Wang, and Hong Jiao, and Young, Michael J. and Brooks, Thomas and Olson, John} } @article {2109, title = {To Weight Or Not To Weight? Balancing Influence Of Initial Items In Adaptive Testing}, journal = {Psychometrica}, volume = {73}, year = {2008}, pages = {441-450}, abstract = {

It has been widely reported that in computerized adaptive testing some examinees may get much lower scores than they would normally if an alternative paper-and-pencil version were given. The main purpose of this investigation is to quantitatively reveal the cause for the underestimation phenomenon. The logistic models, including the 1PL, 2PL, and 3PL models, are used to demonstrate our assertions. Our analytical derivation shows that, under the maximum information item selection strategy, if an examinee failed a few items at the beginning of the test, easy but more discriminating items are likely to be administered. Such items are ineffective to move the estimate close to the true theta, unless the test is sufficiently long or a variable-length test is used. Our results also indicate that a certain weighting mechanism is necessary to make the algorithm rely less on the items administered at the beginning of the test.

}, doi = {10.1007/S11336-007-9047-7}, author = {Chang, H.-H. and Ying, Z.} } @article {51, title = {The Patient-Reported Outcomes Measurement Information System (PROMIS): progress of an NIH Roadmap cooperative group during its first two years}, journal = {Medical Care}, volume = {45}, number = {5 Suppl 1}, year = {2007}, pages = {S3-S11}, abstract = {The National Institutes of Health (NIH) Patient-Reported Outcomes Measurement Information System (PROMIS) Roadmap initiative (www.nihpromis.org) is a 5-year cooperative group program of research designed to develop, validate, and standardize item banks to measure patient-reported outcomes (PROs) relevant across common medical conditions. In this article, we will summarize the organization and scientific activity of the PROMIS network during its first 2 years.}, author = {Cella, D. and Yount, S. and Rothrock, N. and Gershon, R. C. and Cook, K. F. and Reeve, B. and Ader, D. and Fries, J.F. and Bruce, B. and Rose, M.} } @article {2149, title = {Two-Phase Item Selection Procedure for Flexible Content Balancing in CAT}, journal = {Applied Psychological Measurement}, volume = {31}, number = {6}, year = {2007}, pages = {467-482}, abstract = {

Content balancing is an important issue in the design and implementation of computerized adaptive testing (CAT). Content-balancing techniques that have been applied in fixed content balancing, where the number of items from each content area is fixed, include constrained CAT (CCAT), the modified multinomial model (MMM), modified constrained CAT (MCCAT), and others. In this article, four methods are proposed to address the flexible content-balancing issue with the a-stratification design, named STR_C. The four methods are MMM+, an extension of MMM; MCCAT+, an extension of MCCAT; the TPM method, a two-phase content-balancing method using MMM in both phases; and the TPF method, a two-phase content-balancing method using MMM in the first phase and MCCAT in the second. Simulation results show that all of the methods work well in content balancing, and TPF performs the best in item exposure control and item pool utilization while maintaining measurement precision.

}, doi = {10.1177/0146621606292933}, url = {http://apm.sagepub.com/content/31/6/467.abstract}, author = {Ying Cheng, and Chang, Hua-Hua and Qing Yi,} } @article {508, title = {Two-phase item selection procedure for flexible content balancing in CAT}, journal = {Applied Psychological. Measurement}, volume = {3}, year = {2007}, pages = {467{\textendash}482}, author = {Cheng, Y and Chang, Hua-Hua and Yi, Q.} } @article {820, title = {Assessing CAT Test Security Severity}, journal = {Applied Psychological Measurement}, volume = {30(1)}, year = {2006}, pages = {62{\textendash}63}, author = {Yi, Q., Zhang, J. and Chang, Hua-Hua} } @article {247, title = {Comparing methods of assessing differential item functioning in a computerized adaptive testing environment}, journal = {Journal of Educational Measurement}, volume = {43}, number = {3}, year = {2006}, pages = {245-264}, publisher = {Blackwell Publishing: United Kingdom}, abstract = {Mantel-Haenszel and SIBTEST, which have known difficulty in detecting non-unidirectional differential item functioning (DIF), have been adapted with some success for computerized adaptive testing (CAT). This study adapts logistic regression (LR) and the item-response-theory-likelihood-ratio test (IRT-LRT), capable of detecting both unidirectional and non-unidirectional DIF, to the CAT environment in which pretest items are assumed to be seeded in CATs but not used for trait estimation. The proposed adaptation methods were evaluated with simulated data under different sample size ratios and impact conditions in terms of Type I error, power, and specificity in identifying the form of DIF. The adapted LR and IRT-LRT procedures are more powerful than the CAT version of SIBTEST for non-unidirectional DIF detection. The good Type I error control provided by IRT-LRT under extremely unequal sample sizes and large impact is encouraging. Implications of these and other findings are discussed. all rights reserved)}, keywords = {computerized adaptive testing, educational testing, item response theory likelihood ratio test, logistic regression, trait estimation, unidirectional \& non-unidirectional differential item functioning}, isbn = {0022-0655 (Print)}, author = {Lei, P-W. and Chen, S-Y. and Yu, L.} } @article {2198, title = {Comparing Methods of Assessing Differential Item Functioning in a Computerized Adaptive Testing Environment}, journal = {Journal of Educational Measurement}, volume = {43}, number = {3}, year = {2006}, pages = {245{\textendash}264}, abstract = {

Mantel-Haenszel and SIBTEST, which have known difficulty in detecting non-unidirectional differential item functioning (DIF), have been adapted with some success for computerized adaptive testing (CAT). This study adapts logistic regression (LR) and the item-response-theory-likelihood-ratio test (IRT-LRT), capable of detecting both unidirectional and non-unidirectional DIF, to the CAT environment in which pretest items are assumed to be seeded in CATs but not used for trait estimation. The proposed adaptation methods were evaluated with simulated data under different sample size ratios and impact conditions in terms of Type I error, power, and specificity in identifying the form of DIF. The adapted LR and IRT-LRT procedures are more powerful than the CAT version of SIBTEST for non-unidirectional DIF detection. The good Type I error control provided by IRT-LRT under extremely unequal sample sizes and large impact is encouraging. Implications of these and other findings are discussed.

}, issn = {1745-3984}, doi = {10.1111/j.1745-3984.2006.00015.x}, url = {http://dx.doi.org/10.1111/j.1745-3984.2006.00015.x}, author = {Lei, Pui-Wa and Chen, Shu-Ying and Yu, Lan} } @article {2173, title = {Effects of Estimation Bias on Multiple-Category Classification With an IRT-Based Adaptive Classification Procedure}, journal = {Educational and Psychological Measurement}, volume = {66}, number = {4}, year = {2006}, pages = {545-564}, abstract = {

The effects of five ability estimators, that is, maximum likelihood estimator, weighted likelihood estimator, maximum a posteriori, expected a posteriori, and Owen\&$\#$39;s sequential estimator, on the performances of the item response theory\–based adaptive classification procedure on multiple categories were studied via simulations. The following results were found. (a) The Bayesian estimators were more likely to misclassify examinees into an inward category because of their inward biases, when a fixed start value of zero was assigned to every examinee. (b) When moderately accurate start values were available, however, Bayesian estimators produced classifications that were slightly more accurate than was the maximum likelihood estimator or weighted likelihood estimator. Expected a posteriori was the procedure that produced the most accurate results among the three Bayesian methods. (c) All five estimators produced equivalent efficiencies in terms of number of items required, which was 50 or more items except for abilities that were less than -2.00 or greater than 2.00.

}, doi = {10.1177/0013164405284031}, url = {http://epm.sagepub.com/content/66/4/545.abstract}, author = {Yang, Xiangdong and Poggio, John C. and Glasnapp, Douglas R.} } @article {571, title = {Design and evaluation of an XML-based platform-independent computerized adaptive testing system}, journal = {IEEE Transactions on Education}, volume = {48(2)}, year = {2005}, pages = {230-237}, author = {Ho, R.-G., and Yen, Y.-C.} } @conference {1285, title = {Identifying practical indices for enhancing item pool security}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education (NCME)}, year = {2005}, address = {Montreal, Canada}, author = {Yi, Q. and Zhang, J. and Chang, Hua-Hua} } @article {2238, title = {Adaptive Testing With Regression Trees in the Presence of Multidimensionality}, journal = {Journal of Educational and Behavioral Statistics}, volume = {29}, number = {3}, year = {2004}, pages = {293-316}, abstract = {

It is unrealistic to suppose that standard item response theory (IRT) models will be appropriate for all the new and currently considered computer-based tests. In addition to developing new models, we also need to give attention to the possibility of constructing and analyzing new tests without the aid of strong models. Computerized adaptive testing currently relies heavily on IRT. Alternative, empirically based, nonparametric adaptive testing algorithms exist, but their properties are little known. This article introduces a nonparametric, tree-based algorithm for adaptive testing and shows that it may be superior to conventional, IRT-based adaptive testing in cases where the IRT assumptions are not satisfied. In particular, it shows that the tree-based approach clearly outperformed (one-dimensional) IRT when the pool was strongly two-dimensional.

}, doi = {10.3102/10769986029003293}, url = {http://jeb.sagepub.com/cgi/content/abstract/29/3/293}, author = {Yan, Duanli and Lewis, Charles and Stocking, Martha} } @article {381, title = {Pre-equating: a simulation study based on a large scale assessment model}, journal = {Journal of Applied Measurement}, volume = {5}, number = {3}, year = {2004}, note = {1529-7713Journal Article}, pages = {301-18}, abstract = {Although post-equating (PE) has proven to be an acceptable method in the scaling and equating of items and forms, there are times when the turn-around period for equating and converting raw scores to scale scores is so small that PE cannot be undertaken within the prescribed time frame. In such cases, pre-equating (PrE) could be considered as an acceptable alternative. Assessing the feasibility of using item calibrations from the item bank (as in PrE) is conditioned on the equivalency of the calibrations and the errors associated with it vis a vis the results obtained via PE. This paper creates item banks over three periods of item introduction into the banks and uses the Rasch model in examining data with respect to the recovery of item parameters, the measurement error, and the effect cut-points have on examinee placement in both the PrE and PE situations. Results indicate that PrE is a viable solution to PE provided the stability of the item calibrations are enhanced by using large sample sizes (perhaps as large as full-population) in populating the item bank.}, keywords = {*Databases, *Models, Theoretical, Calibration, Human, Psychometrics, Reference Values, Reproducibility of Results}, author = {Taherbhai, H. M. and Young, M. J.} } @article {2133, title = {a-Stratified multistage CAT design with content-blocking}, journal = {British Journal of Mathematical and Statistical Psychology}, volume = {56}, year = {2003}, pages = {359{\textendash}378}, author = {Yi, Q. and Chang, H.-H.} } @conference {1280, title = {Effects of test administration mode on item parameter estimates}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {2003}, note = {{PDF file, 233 KB}}, address = {Chicago IL}, author = {Yi, Q. and Harris, D. J. and Wang, T. and Ban, J-C.} } @conference {1283, title = {Implementing the a-stratified method with b blocking in computerized adaptive testing with the generalized partial credit model}, booktitle = {Paper presented at the annual meeting of the American Educational Research Association}, year = {2003}, note = {$\#$YI03-01 {PDF file, 496 KB}}, address = {Chicago IL}, author = {Yi, Q. and Wang, T. and Wang, S} } @conference {874, title = {Test-score comparability, ability estimation, and item-exposure control in computerized adaptive testing}, booktitle = {Paper presented at the Annual meeting of the National Council on Measurement in Education}, year = {2003}, address = {Chicago IL}, author = {Chang, Hua-Hua and Ying, Z.} } @conference {1015, title = {Accuracy of the ability estimate and the item exposure rate under multidimensional adaptive testing with item constraints}, booktitle = {Paper presented at the annual meeting of the American Educational Research Association}, year = {2002}, note = {$\#$LI02-01}, address = {New Orleans LA}, author = {Li, Y. H. and Yu, N. Y.} } @booklet {1583, title = {Adaptive testing without IRT in the presence of multidimensionality (Research Report 02-09)}, year = {2002}, address = {Princeton NJ: Educational Testing Service}, author = {Yan, D. and Lewis, C. and Stocking, M.} } @article {14, title = {Data sparseness and on-line pretest item calibration-scaling methods in CAT}, journal = {Journal of Educational Measurement}, volume = {39}, number = {3}, year = {2002}, pages = {207-218}, abstract = {Compared and evaluated 3 on-line pretest item calibration-scaling methods (the marginal maximum likelihood estimate with 1 expectation maximization [EM] cycle [OEM] method, the marginal maximum likelihood estimate with multiple EM cycles [MEM] method, and M. L. Stocking{\textquoteright}s Method B) in terms of item parameter recovery when the item responses to the pretest items in the pool are sparse. Simulations of computerized adaptive tests were used to evaluate the results yielded by the three methods. The MEM method produced the smallest average total error in parameter estimation, and the OEM method yielded the largest total error (PsycINFO Database Record (c) 2005 APA )}, keywords = {Computer Assisted Testing, Educational Measurement, Item Response Theory, Maximum Likelihood, Methodology, Scaling (Testing), Statistical Data}, author = {Ban, J-C. and Hanson, B. A. and Yi, Q. and Harris, D. J.} } @conference {1277, title = {Incorporating the Sympson-Hetter exposure control method into the a-stratified method with content blocking}, booktitle = {Paper presented at the annual meeting of the American Educational Research Association}, year = {2002}, note = {PDF file, 387 K}, address = {New Orleans, LA}, author = {Yi, Q.} } @conference {1294, title = {Statistical indexes for monitoring item behavior under computer adaptive testing environment}, booktitle = {(Original title: Detecting item misfit in computerized adaptive testing.) Paper presented at the annual meeting of the American Educational Research Association}, year = {2002}, note = {{PDF file, 2.287 MB}}, address = {New Orleans LA}, author = {Zhu, R. and Yu, F. and Liu, S. M.} } @conference {873, title = {To weight or not to weight {\textendash} balancing influence of initial and later items in CAT}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {2002}, note = {{PDF file, 252 KB}}, address = {New Orleans LA}, author = {Chang, Hua-Hua and Ying, Z.} } @conference {1278, title = {a-stratified CAT design with content-blocking}, booktitle = {Paper presented at the Annual Meeting of the Psychometric Society}, year = {2001}, note = {{PDF file, 410 KB}}, address = {King of Prussia, PA}, author = {Yi, Q. and Chang, Hua-Hua} } @article {55, title = {a-stratified multistage computerized adaptive testing with b blocking}, journal = {Applied Psychological Measurement}, volume = {25}, number = {4}, year = {2001}, pages = {333-41}, abstract = {Proposed a refinement, based on the stratification of items developed by D. Weiss (1973), of the computerized adaptive testing item selection procedure of H. Chang and Z. Ying (1999). Simulation studies using an item bank from the Graduate Record Examination show the benefits of the new procedure. (SLD)}, keywords = {computerized adaptive testing}, author = {Chang, Hua-Hua and Qian, J. and Yang, Z.} } @article {56, title = {a-Stratified multistage computerized adaptive testing with b blocking}, journal = {Applied Psychological Measurement}, volume = {25}, number = {4}, year = {2001}, pages = {333-341}, abstract = {Chang \& Ying{\textquoteright}s (1999) computerized adaptive testing item-selection procedure stratifies the item bank according to a parameter values and requires b parameter values to be evenly distributed across all strata. Thus, a and b parameter values must be incorporated into how strata are formed. A refinement is proposed, based on Weiss{\textquoteright} (1973) stratification of items according to b values. Simulation studies using a retired item bank of a Graduate Record Examination test indicate that the new approach improved control of item exposure rates and reduced mean squared errors. }, isbn = {0146-6216}, author = {Chang, Hua-Hua and Qian, J. and Ying, Z.} } @article {13, title = {A comparative study of on line pretest item{\textemdash}Calibration/scaling methods in computerized adaptive testing}, journal = {Journal of Educational Measurement}, volume = {38}, number = {3}, year = {2001}, pages = {191-212}, abstract = {The purpose of this study was to compare and evaluate five on-line pretest item-calibration/scaling methods in computerized adaptive testing (CAT): marginal maximum likelihood estimate with one EM cycle (OEM), marginal maximum likelihood estimate with multiple EM cycles (MEM), Stocking{\textquoteright}s Method A, Stocking{\textquoteright}s Method B, and BILOG/Prior. The five methods were evaluated in terms ofitem-parameter recovery, using three different sample sizes (300, 1000 and 3000). The MEM method appeared to be the best choice among these, because it produced the smallest parameter-estimation errors for all sample size conditions. MEM and OEM are mathematically similar, although the OEM method produced larger errors. MEM also was preferable to OEM, unless the amount of timeinvolved in iterative computation is a concern. Stocking{\textquoteright}s Method B also worked very well, but it required anchor items that either would increase test lengths or require larger sample sizes depending on test administration design. Until more appropriate ways of handling sparse data are devised, the BILOG/Prior method may not be a reasonable choice for small sample sizes. Stocking{\textquoteright}s Method A hadthe largest weighted total error, as well as a theoretical weakness (i.e., treating estimated ability as true ability); thus, there appeared to be little reason to use it}, author = {Ban, J. C. and Hanson, B. A. and Wang, T. and Yi, Q. and Harris, D. J.} } @conference {1279, title = {Comparison of the SPRT and CMT procedures in computerized adaptive testing}, booktitle = {Paper presented at the annual meeting of the American Educational Research Association}, year = {2001}, address = {Seattle WA}, author = {Yi, Q. and Hanson, B. and Widiatmo, H. and Harris, D. J.} } @conference {835, title = {Data sparseness and online pretest calibration/scaling methods in CAT}, booktitle = {Paper presented at the annual meeting of the American Educational Research Association}, year = {2001}, note = {Also ACT Research Report 2002-1)}, address = {Seattle}, author = {Ban, J and Hanson, B. A. and Yi, Q. and Harris, D.} } @conference {930, title = {Deriving a stopping rule for sequential adaptive tests}, booktitle = {Paper presented at the annual meeting of the American Educational Research Association}, year = {2001}, note = {{PDF file, 111 KB}}, address = {Seattle WA}, author = {Grabovsky, I. and Chang, Hua-Hua and Ying, Z.} } @conference {1284, title = {Impact of scoring options for not reached items in CAT}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {2001}, note = {PDF file, 232 K}, address = {Seattle WA}, author = {Yi, Q. and Widiatmo, H. and Ban, J-C. and Harris, D. J.} } @article {482, title = {On maximizing item information and matching difficulty with ability}, journal = {Psychometrika}, volume = {66}, year = {2001}, pages = {69-77}, author = {Bickel, P. and Buyske, S. and Chang, Hua-Hua and Ying, Z.} } @conference {837, title = {Effects of nonequivalence of item pools on ability estimates in CAT}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {2000}, note = {PDF file, 657 K}, address = {New Orleans LA}, author = {Ban, J. C. and Wang, T. and Yi, Q. and Harris, D. J.} } @booklet {1584, title = {Multiple stratification CAT designs with content control}, year = {2000}, address = {Unpublished manuscript}, author = {Yi, Q. and Chang, Hua-Hua} } @conference {1244, title = {Adjusting "scores" from a CAT following successful item challenges}, booktitle = {Paper presented at the annual meeting of the American Educational Research Association}, year = {1999}, note = {$\#$WA99-01 {PDF file, 150 KB}}, address = {Montreal, Canada}, author = {Wang, T. and Yi, Q. and Ban, J. C. and Harris, D. J. and Hanson, B. A.} } @article {59, title = {a-stratified multistage computerized adaptive testing}, journal = {Applied Psychological Measurement}, volume = {23}, number = {3}, year = {1999}, note = {Sage Publications, US}, pages = {211-222}, abstract = {For computerized adaptive tests (CAT) based on the three-parameter logistic mode it was found that administering items with low discrimination parameter (a) values early in the test and administering those with high a values later was advantageous; the skewness of item exposure distributions was reduced while efficiency was maintain in trait level estimation. Thus, a new multistage adaptive testing approach is proposed that factors a into the item selection process. In this approach, the items in the item bank are stratified into a number of levels based on their a values. The early stages of a test use items with lower as and later stages use items with higher as. At each stage, items are selected according to an optimization criterion from the corresponding level. Simulation studies were performed to compare a-stratified CATs with CATs based on the Sympson-Hetter method for controlling item exposure. Results indicated that this new strategy led to tests that were well-balanced, with respect to item exposure, and efficient. The a-stratified CATs achieved a lower average exposure rate than CATs based on Bayesian or information-based item selection and the Sympson-Hetter method. (PsycINFO Database Record (c) 2003 APA, all rights reserved).}, keywords = {computerized adaptive testing}, author = {Chang, Hua-Hua and Ying, Z.} } @article {497, title = {a-stratified multistage computerized adaptive testing}, journal = {Applied Psychological Measurement}, volume = {23}, year = {1999}, pages = {211-222}, author = {Chang, Hua-Hua and Ying, Z.} } @article {495, title = {a-stratified multistage computerized adaptive testing}, journal = {Applied Psychological Measurement}, volume = {23}, year = {1999}, pages = {211{\textendash}222}, author = {Chang, Hua-Hua and Ying, Z.} } @conference {836, title = {Comparison of the a-stratified method, the Sympson-Hetter method, and the matched difficulty method in CAT administration}, booktitle = {Paper presented at the annual meeting of the Psychometric Society}, year = {1999}, note = {$\#$BA99-01}, address = {Lawrence KS}, author = {Ban, J and Wang, T. and Yi, Q.} } @conference {1087, title = {Performance of the Sympson-Hetter exposure control algorithm with a polytomous item bank}, booktitle = {Paper presented at the annual meeting of American Educational Research Association}, year = {1999}, address = {Montreal, Canada}, author = {Pastor, D. A. and Chiang, C. and Dodd, B. G. and Yockey, R. and} } @conference {1276, title = {Adaptive testing without IRT}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1998}, note = {(ERIC No. ED422359)}, address = {San Diego CA}, author = {Yan, D. and Lewis, C. and Stocking, M.} } @conference {1282, title = {The impact of nonmodel-fitting responses in a realistic CAT environment}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1998}, address = {San Diego CA}, author = {Yi, Q, and Nering, M.} } @conference {1281, title = {Nonmodel-fitting responses and robust ability estimation in a realistic CAT environment}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1998}, address = {San Diego CA}, author = {Yi, Q, and Nering, M.} } @booklet {1585, title = {Simulating nonmodel-fitting responses in a CAT Environment (Research Report 98-10)}, year = {1998}, note = {$\#$YI-98-10}, address = {Iowa City IA: ACT Inc. (Also presented at National Council on Measurement in Education, 1999: ERIC No. ED 427 042)}, author = {Yi, Q. and Nering, M, L.} } @conference {872, title = {Multi-stage CAT with stratified design}, booktitle = {Paper presented at the annual meeting of the Psychometric Society. Gatlinburg TN.}, year = {1997}, author = {Chang, Hua-Hua and Ying, Z.} } @article {496, title = {Nonlinear sequential designs for logistic item response theory models with applications to computerized adaptive tests}, journal = {The Annals of Statistics.}, year = {1997}, author = {Chang, Hua-Hua and Ying, Z.} } @conference {871, title = {Building a statistical foundation for computerized adaptive testing}, booktitle = {Paper presented at the annual meeting of the Psychometric Society}, year = {1996}, address = {Banff, Alberta, Canada}, author = {Chang, Hua-Hua and Ying, Z.} } @article {686, title = {Conducting self-adapted testing using MicroCAT}, journal = {Educational and Psychological Measurement}, volume = {56}, year = {1996}, pages = {821-827}, author = {Roos, L. L. and Wise, S. L. and Yoes, M. E. and Rocklin, T. R.} } @article {58, title = {A global information approach to computerized adaptive testing}, journal = {Applied Psychological Measurement}, volume = {20}, number = {3}, year = {1996}, pages = {213-229}, abstract = {based on Fisher information (or item information). At each stage, an item is selected to maximize the Fisher information at the currently estimated trait level (\&thetas;). However, this application of Fisher information could be much less efficient than assumed if the estimators are not close to the true \&thetas;, especially at early stages of an adaptive test when the test length (number of items) is too short to provide an accurate estimate for true \&thetas;. It is argued here that selection procedures based on global information should be used, at least at early stages of a test when \&thetas; estimates are not likely to be close to the true \&thetas;. For this purpose, an item selection procedure based on average global information is proposed. Results from pilot simulation studies comparing the usual maximum item information item selection with the proposed global information approach are reported, indicating that the new method leads to improvement in terms of bias and mean squared error reduction under many circumstances. Index terms: computerized adaptive testing, Fisher information, global information, information surface, item information, item response theory, Kullback-Leibler information, local information, test information.}, isbn = {0146-6216}, author = {Chang, Hua-Hua and Ying, Z.} } @article {2032, title = {A Global Information Approach to Computerized Adaptive Testing}, journal = {Applied Psychological Measurement}, volume = {20}, year = {1996}, pages = {213-229}, author = {Chang, H.-H. and Ying, Z.} } @booklet {1346, title = {Recursive maximum likelihood estimation, sequential design, and computerized adaptive testing}, year = {1996}, address = {Princeton NJ: Educational Testing Service}, author = {Chang, Hua-Hua and Ying, Z.} } @conference {1286, title = {Recursive maximum likelihood estimation, sequential designs, and computerized adaptive testing}, booktitle = {Paper presented at the Eleventh Workshop on Item Response Theory}, year = {1995}, address = {University of Twente, the Netherlands}, author = {Ying, Z. and Chang, Hua-Hua} } @conference {1287, title = {Methods of selecting successive items in adaptive testing}, booktitle = {Unpublished manuscript}, year = {1985}, address = {University of Pittsburgh}, author = {Yu, L.} }