@article {2738, title = {Routing Strategies and Optimizing Design for Multistage Testing in International Large-Scale Assessments}, journal = {Journal of Educational Measurement}, volume = {56}, number = {1}, year = {2019}, pages = {192-213}, abstract = {Abstract This study investigates the effect of several design and administration choices on item exposure and person/item parameter recovery under a multistage test (MST) design. In a simulation study, we examine whether number-correct (NC) or item response theory (IRT) methods are differentially effective at routing students to the correct next stage(s) and whether routing choices (optimal versus suboptimal routing) have an impact on achievement precision. Additionally, we examine the impact of testlet length on both person and item recovery. Overall, our results suggest that no single approach works best across the studied conditions. With respect to the mean person parameter recovery, IRT scoring (via either Fisher information or preliminary EAP estimates) outperformed classical NC methods, although differences in bias and root mean squared error were generally small. Item exposure rates were found to be more evenly distributed when suboptimal routing methods were used, and item recovery (both difficulty and discrimination) was most precisely observed for items with moderate difficulties. Based on the results of the simulation study, we draw conclusions and discuss implications for practice in the context of international large-scale assessments that recently introduced adaptive assessment in the form of MST. Future research directions are also discussed.}, doi = {10.1111/jedm.12206}, url = {https://onlinelibrary.wiley.com/doi/abs/10.1111/jedm.12206}, author = {Svetina, Dubravka and Liaw, Yuan-Ling and Rutkowski, Leslie and Rutkowski, David} } @conference {2670, title = {Response Time and Response Accuracy in Computerized Adaptive Testing}, booktitle = {IACAT 2017 Conference}, year = {2017}, month = {08/2017}, publisher = {Niigata Seiryo University}, organization = {Niigata Seiryo University}, address = {Niigata, Japan}, abstract = {

Introduction. This study explores the relationship between response speed and response accuracy in Computerized Adaptive Testing (CAT). CAT provides a score as well as item response times, which can offer additional diagnostic information regarding behavioral processes of task completion that cannot be uncovered by paper-based instruments. The goal of this study is to investigate how the accuracy rate evolves as a function of response time. If the accuracy of cognitive test responses decreases with response time, then it is an indication that the underlying cognitive process is a degrading process such as knowledge retrieval. More accessible knowledge can be retrieved faster than less accessible knowledge. For instance, in reading tasks, the time on task effect is negative and the more negative, the easier a task is. However, if the accuracy of cognitive test responses increases with response time, then the process is of an upgrading nature, with an increasing success rate as a function of response time. For example, problem-solving takes time, and fast responses are less likely to be well-founded responses. It is of course also possible that the relationship is curvilinear, as when an increasing success rate is followed by a decreasing success rate or vice versa.

Hypothesis. The present study argues the relationship between response time on task and response accuracy can be positive, negative, or curvilinear, which depends on cognitive nature of task items holding ability of the subjects and difficulty of the items constant.

Methodology. Data from a subsection of GRE quantitative test were available. We will use generalized linear mixed models. A linear model means a linear combination of predictors determining the probability of person p for answering item i correctly. Modeling mixed effects means both random effects and fixed effects are included. Fixed effects refer to constants across test takers. The models are equivalent with advanced IRT models that go beyond the regular modeling of test responses in terms of one or more latent variables and item parameters. The lme4 package for R will be utilized to conduct the statistical calculation.

Research questions. 1. What is the relationship between response accuracy and response speed? 2. What is the correlation between response accuracy and type of response time (fast response vs slow response) after controlling ability of people?

Preliminary Findings. 1. There is a negative relationship between response time and response accuracy. The success rate declines with elapsing response time. 2. The correlation between the two response latent variables (fast and slow) is 1.0, indicating the time on task effects between respond time types are not different.

Implications. The right amount of testing time in CAT is important\—too much is wasteful and costly, too little impacts score validity. The study is expected to provide new perception on the relationship between response time and response accuracy, which in turn, contribute to the best timing strategy in CAT\—with or without time constraints.

Session Video

}, keywords = {CAT, response accuracy, Response time}, url = {https://drive.google.com/open?id=1yYP01bzGrKvJnfLwepcAoQQ2F4TdSvZ2}, author = {Yang Shi} } @article {2613, title = {Robust Automated Test Assembly for Testlet-Based Tests: An Illustration with Analytical Reasoning Items}, journal = {Frontiers in Education}, volume = {2}, year = {2017}, pages = {63}, issn = {2504-284X}, doi = {10.3389/feduc.2017.00063}, url = {https://www.frontiersin.org/article/10.3389/feduc.2017.00063}, author = {Veldkamp, Bernard P. and Paap, Muirne C. S.} } @article {2283, title = {The Random-Threshold Generalized Unfolding Model and Its Application of Computerized Adaptive Testing}, journal = {Applied Psychological Measurement}, volume = {37}, number = {3}, year = {2013}, pages = {179-200}, abstract = {

The random-threshold generalized unfolding model (RTGUM) was developed by treating the thresholds in the generalized unfolding model as random effects rather than fixed effects to account for the subjective nature of the selection of categories in Likert items. The parameters of the new model can be estimated with the JAGS (Just Another Gibbs Sampler) freeware, which adopts a Bayesian approach for estimation. A series of simulations was conducted to evaluate the parameter recovery of the new model and the consequences of ignoring the randomness in thresholds. The results showed that the parameters of RTGUM were recovered fairly well and that ignoring the randomness in thresholds led to biased estimates. Computerized adaptive testing was also implemented on RTGUM, where the Fisher information criterion was used for item selection and the maximum a posteriori method was used for ability estimation. The simulation study showed that the longer the test length, the smaller the randomness in thresholds, and the more categories in an item, the more precise the ability estimates would be.

}, doi = {10.1177/0146621612469720}, url = {http://apm.sagepub.com/content/37/3/179.abstract}, author = {Wang, Wen-Chung and Liu, Chen-Wei and Wu, Shiu-Lien} } @inbook {2273, title = {Reporting differentiated literacy results in PISA by using multidimensional adaptive testing. }, booktitle = {Research on PISA.}, year = {2013}, publisher = {Dodrecht: Springer}, organization = {Dodrecht: Springer}, author = {Frey, A. and Seitz, N-N. and Kr{\"o}hne, U.} } @article {2194, title = {On the Reliability and Validity of a Numerical Reasoning Speed Dimension Derived From Response Times Collected in Computerized Testing}, journal = {Educational and Psychological Measurement}, volume = {72}, number = {2}, year = {2012}, pages = {245-263}, abstract = {

Data from 181 college students were used to assess whether math reasoning item response times in computerized testing can provide valid and reliable measures of a speed dimension. The alternate forms reliability of the speed dimension was .85. A two-dimensional structural equation model suggests that the speed dimension is related to the accuracy of speeded responses. Speed factor scores were significantly correlated with performance on the ACT math scale. Results suggest that the speed dimension underlying response times can be reliably measured and that the dimension is related to the accuracy of performance under the pressure of time limits.

}, doi = {10.1177/0013164411408412}, url = {http://epm.sagepub.com/content/72/2/245.abstract}, author = {Davison, Mark L. and Semmes, Robert and Huang, Lan and Close, Catherine N.} } @article {2204, title = {Restrictive Stochastic Item Selection Methods in Cognitive Diagnostic Computerized Adaptive Testing}, journal = {Journal of Educational Measurement}, volume = {48}, number = {3}, year = {2011}, pages = {255{\textendash}273}, abstract = {

This paper proposes two new item selection methods for cognitive diagnostic computerized adaptive testing: the restrictive progressive method and the restrictive threshold method. They are built upon the posterior weighted Kullback-Leibler (KL) information index but include additional stochastic components either in the item selection index or in the item selection procedure. Simulation studies show that both methods are successful at simultaneously suppressing overexposed items and increasing the usage of underexposed items. Compared to item selection based upon (1) pure KL information and (2) the Sympson-Hetter method, the two new methods strike a better balance between item exposure control and measurement accuracy. The two new methods are also compared with Barrada et al.\&$\#$39;s (2008) progressive method and proportional method.

}, issn = {1745-3984}, doi = {10.1111/j.1745-3984.2011.00145.x}, url = {http://dx.doi.org/10.1111/j.1745-3984.2011.00145.x}, author = {Wang, Chun and Chang, Hua-Hua and Huebner, Alan} } @article {78, title = {Reduction in patient burdens with graphical computerized adaptive testing on the ADL scale: tool development and simulation}, journal = {Health and Quality of Life Outcomes}, volume = {7}, year = {2009}, note = {Chien, Tsair-WeiWu, Hing-ManWang, Weng-ChungCastillo, Roberto VasquezChou, WillyComparative StudyValidation StudiesEnglandHealth and quality of life outcomesHealth Qual Life Outcomes. 2009 May 5;7:39.}, pages = {39}, edition = {2009/05/07}, abstract = {BACKGROUND: The aim of this study was to verify the effectiveness and efficacy of saving time and reducing burden for patients, nurses, and even occupational therapists through computer adaptive testing (CAT). METHODS: Based on an item bank of the Barthel Index (BI) and the Frenchay Activities Index (FAI) for assessing comprehensive activities of daily living (ADL) function in stroke patients, we developed a visual basic application (VBA)-Excel CAT module, and (1) investigated whether the averaged test length via CAT is shorter than that of the traditional all-item-answered non-adaptive testing (NAT) approach through simulation, (2) illustrated the CAT multimedia on a tablet PC showing data collection and response errors of ADL clinical functional measures in stroke patients, and (3) demonstrated the quality control of endorsing scale with fit statistics to detect responding errors, which will be further immediately reconfirmed by technicians once patient ends the CAT assessment. RESULTS: The results show that endorsed items could be shorter on CAT (M = 13.42) than on NAT (M = 23) at 41.64\% efficiency in test length. However, averaged ability estimations reveal insignificant differences between CAT and NAT. CONCLUSION: This study found that mobile nursing services, placed at the bedsides of patients could, through the programmed VBA-Excel CAT module, reduce the burden to patients and save time, more so than the traditional NAT paper-and-pencil testing appraisals.}, keywords = {*Activities of Daily Living, *Computer Graphics, *Computer Simulation, *Diagnosis, Computer-Assisted, Female, Humans, Male, Point-of-Care Systems, Reproducibility of Results, Stroke/*rehabilitation, Taiwan, United States}, isbn = {1477-7525 (Electronic)1477-7525 (Linking)}, author = {Chien, T. W. and Wu, H. M. and Wang, W-C. and Castillo, R. V. and Chou, W.} } @article {173, title = {Replenishing a computerized adaptive test of patient-reported daily activity functioning}, journal = {Quality of Life Research}, volume = {18}, number = {4}, year = {2009}, note = {Haley, Stephen MNi, PengshengJette, Alan MTao, WeiMoed, RichardMeyers, DougLudlow, Larry HK02 HD45354-01/HD/NICHD NIH HHS/United StatesResearch Support, N.I.H., ExtramuralNetherlandsQuality of life research : an international journal of quality of life aspects of treatment, care and rehabilitationQual Life Res. 2009 May;18(4):461-71. Epub 2009 Mar 14.}, month = {May}, pages = {461-71}, edition = {2009/03/17}, abstract = {PURPOSE: Computerized adaptive testing (CAT) item banks may need to be updated, but before new items can be added, they must be linked to the previous CAT. The purpose of this study was to evaluate 41 pretest items prior to including them into an operational CAT. METHODS: We recruited 6,882 patients with spine, lower extremity, upper extremity, and nonorthopedic impairments who received outpatient rehabilitation in one of 147 clinics across 13 states of the USA. Forty-one new Daily Activity (DA) items were administered along with the Activity Measure for Post-Acute Care Daily Activity CAT (DA-CAT-1) in five separate waves. We compared the scoring consistency with the full item bank, test information function (TIF), person standard errors (SEs), and content range of the DA-CAT-1 to the new CAT (DA-CAT-2) with the pretest items by real data simulations. RESULTS: We retained 29 of the 41 pretest items. Scores from the DA-CAT-2 were more consistent (ICC = 0.90 versus 0.96) than DA-CAT-1 when compared with the full item bank. TIF and person SEs were improved for persons with higher levels of DA functioning, and ceiling effects were reduced from 16.1\% to 6.1\%. CONCLUSIONS: Item response theory and online calibration methods were valuable in improving the DA-CAT.}, keywords = {*Activities of Daily Living, *Disability Evaluation, *Questionnaires, *User-Computer Interface, Adult, Aged, Cohort Studies, Computer-Assisted Instruction, Female, Humans, Male, Middle Aged, Outcome Assessment (Health Care)/*methods}, isbn = {0962-9343 (Print)0962-9343 (Linking)}, author = {Haley, S. M. and Ni, P. and Jette, A. M. and Tao, W. and Moed, R. and Meyers, D. and Ludlow, L. H.} } @article {17, title = {Rotating item banks versus restriction of maximum exposure rates in computerized adaptive testing}, journal = {Spanish Journal of Psychology}, volume = {11}, number = {2}, year = {2008}, note = {Barrada, Juan RamonOlea, JulioAbad, Francisco JoseResearch Support, Non-U.S. Gov{\textquoteright}tSpainThe Spanish journal of psychologySpan J Psychol. 2008 Nov;11(2):618-25.}, pages = {618-625}, edition = {2008/11/08}, abstract = {

If examinees were to know, beforehand, part of the content of a computerized adaptive test, their estimated trait levels would then have a marked positive bias. One of the strategies to avoid this consists of dividing a large item bank into several sub-banks and rotating the sub-bank employed (Ariel, Veldkamp \& van der Linden, 2004). This strategy permits substantial improvements in exposure control at little cost to measurement accuracy, However, we do not know whether this option provides better results than using the master bank with greater restriction in the maximum exposure rates (Sympson \& Hetter, 1985). In order to investigate this issue, we worked with several simulated banks of 2100 items, comparing them, for RMSE and overlap rate, with the same banks divided in two, three... up to seven sub-banks. By means of extensive manipulation of the maximum exposure rate in each bank, we found that the option of rotating banks slightly outperformed the option of restricting maximum exposure rate of the master bank by means of the Sympson-Hetter method.

}, keywords = {*Character, *Databases, *Software Design, Aptitude Tests/*statistics \& numerical data, Bias (Epidemiology), Computing Methodologies, Diagnosis, Computer-Assisted/*statistics \& numerical data, Educational Measurement/*statistics \& numerical data, Humans, Mathematical Computing, Psychometrics/statistics \& numerical data}, isbn = {1138-7416}, author = {Barrada, J and Olea, J. and Abad, F. J.} } @article {333, title = {Relative precision, efficiency and construct validity of different starting and stopping rules for a computerized adaptive test: The GAIN Substance Problem Scale}, journal = {Journal of Applied Measurement}, volume = {8}, number = {1}, year = {2007}, pages = {48-65}, abstract = {Substance abuse treatment programs are being pressed to measure and make clinical decisions more efficiently about an increasing array of problems. This computerized adaptive testing (CAT) simulation examined the relative efficiency, precision and construct validity of different starting and stopping rules used to shorten the Global Appraisal of Individual Needs{\textquoteright} (GAIN) Substance Problem Scale (SPS) and facilitate diagnosis based on it. Data came from 1,048 adolescents and adults referred to substance abuse treatment centers in 5 sites. CAT performance was evaluated using: (1) average standard errors, (2) average number of items, (3) bias in personmeasures, (4) root mean squared error of person measures, (5) Cohen{\textquoteright}s kappa to evaluate CAT classification compared to clinical classification, (6) correlation between CAT and full-scale measures, and (7) construct validity of CAT classification vs. clinical classification using correlations with five theoretically associated instruments. Results supported both CAT efficiency and validity.}, keywords = {My article}, author = {Riley, B. B. and Conrad, K. J. and Bezruczko, N. and Dennis, M. L.} } @article {555, title = {A review of item exposure control strategies for computerized adaptive testing developed from 1983 to 2005}, journal = {Journal of Technology,Learning, and Assessment,}, volume = { 5(8)}, year = {2007}, note = {http://www.jtla.org. {PDF file, 326 KB}}, abstract = {Since researchers acknowledged the several advantages of computerized adaptive testing (CAT) over traditional linear test administration, the issue of item exposure control has received increased attention. Due to CAT{\textquoteright}s underlying philosophy, particular items in the item pool may be presented too often and become overexposed, while other items are rarely selected by the CAT algorithm and thus become underexposed. Several item exposure control strategies have been presented in the literature aiming to prevent overexposure of some items and to increase the use rate of rarely or never selected items. This paper reviews such strategies that appeared in the relevant literature from 1983 to 2005. The focus of this paper is on studies that have been conducted in order to evaluate the effectiveness of item exposure control strategies for dichotomous scoring, polytomous scoring and testlet-based CAT systems. In addition, the paper discusses the strengths and weaknesses of each strategy group using examples from simulation studies. No new research is presented but rather a compendium of models is reviewed with an overall objective of providing researchers of this field, especially newcomers, a wide view of item exposure control strategies.}, author = {Georgiadou, E. and Triantafillou, E. and Economides, A. A.} } @article {198, title = {A randomized experiment to compare conventional, computerized, and computerized adaptive administration of ordinal polytomous attitude items}, journal = {Applied Psychological Measurement}, volume = {29}, number = {3}, year = {2005}, pages = {159-183}, abstract = {A total of 520 high school students were randomly assigned to a paper-and-pencil test (PPT), a computerized standard test (CST), or a computerized adaptive test (CAT) version of the Dutch School Attitude Questionnaire (SAQ), consisting of ordinal polytomous items. The CST administered items in the same order as the PPT. The CAT administered all items of three SAQ subscales in adaptive order using Samejima{\textquoteright}s graded response model, so that six different stopping rule settings could be applied afterwards. School marks were used as external criteria. Results showed significant but small multivariate administration mode effects on conventional raw scores and small to medium effects on maximum likelihood latent trait estimates. When the precision of CAT latent trait estimates decreased, correlations with grade point average in general decreased. However, the magnitude of the decrease was not very large as compared to the PPT, the CST, and the CAT without the stopping rule. (PsycINFO Database Record (c) 2005 APA ) (journal abstract)}, keywords = {Computer Assisted Testing, Test Administration, Test Items}, author = {Hol, A. M. and Vorst, H. C. M. and Mellenbergh, G. J.} } @article {2140, title = {A Randomized Experiment to Compare Conventional, Computerized, and Computerized Adaptive Administration of Ordinal Polytomous Attitude Items}, journal = {Applied Psychological Measurement}, volume = {29}, number = {3}, year = {2005}, pages = {159-183}, abstract = {

A total of 520 high school students were randomly assigned to a paper-and-pencil test (PPT), a computerized standard test (CST), or a computerized adaptive test (CAT) version of the Dutch School Attitude Questionnaire (SAQ), consisting of ordinal polytomous items. The CST administered items in the same order as the PPT. The CAT administered all items of three SAQ subscales in adaptive order using Samejima\’s graded response model, so that six different stopping rule settings could be applied afterwards. School marks were used as external criteria. Results showed significant but small multivariate administration mode effects on conventional raw scores and small to medium effects on maximum likelihood latent trait estimates. When the precision of CAT latent trait estimates decreased, correlations with grade point average in general decreased. However, the magnitude of the decrease was not very large as compared to the PPT, the CST, and the CAT without the stopping rule.

}, doi = {10.1177/0146621604271268}, url = {http://apm.sagepub.com/content/29/3/159.abstract}, author = {Hol, A. Michiel and Vorst, Harrie C. M. and Mellenbergh, Gideon J.} } @article {304, title = {Recent trends in comparability studies}, number = {05-05}, year = {2005}, month = {August, 2005}, institution = {Pearson}, keywords = {computer adaptive testing, Computerized assessment, differential item functioning, Mode effects}, isbn = {05-05}, author = {Paek, P.} } @conference {2135, title = {Rescuing CAT by fixing the problems}, booktitle = {National Council on Measurement in Education}, year = {2005}, address = {Montreal, Canada}, author = {Chang, S-H. and Zhang, J.} } @article {87, title = {Refining the conceptual basis for rehabilitation outcome measurement: personal care and instrumental activities domain}, journal = {Medical Care}, volume = {42}, number = {1 Suppl}, year = {2004}, note = {0025-7079Journal Article}, month = {Jan}, pages = {I62-172}, abstract = {BACKGROUND: Rehabilitation outcome measures routinely include content on performance of daily activities; however, the conceptual basis for item selection is rarely specified. These instruments differ significantly in format, number, and specificity of daily activity items and in the measurement dimensions and type of scale used to specify levels of performance. We propose that a requirement for upper limb and hand skills underlies many activities of daily living (ADL) and instrumental activities of daily living (IADL) items in current instruments, and that items selected based on this definition can be placed along a single functional continuum. OBJECTIVE: To examine the dimensional structure and content coverage of a Personal Care and Instrumental Activities item set and to examine the comparability of items from existing instruments and a set of new items as measures of this domain. METHODS: Participants (N = 477) from 3 different disability groups and 4 settings representing the continuum of postacute rehabilitation care were administered the newly developed Activity Measure for Post-Acute Care (AM-PAC), the SF-8, and an additional setting-specific measure: FIM (in-patient rehabilitation); MDS (skilled nursing facility); MDS-PAC (postacute settings); OASIS (home care); or PF-10 (outpatient clinic). Rasch (partial-credit model) analyses were conducted on a set of 62 items covering the Personal Care and Instrumental domain to examine item fit, item functioning, and category difficulty estimates and unidimensionality. RESULTS: After removing 6 misfitting items, the remaining 56 items fit acceptably along the hypothesized continuum. Analyses yielded different difficulty estimates for the maximum score (eg, "Independent performance") for items with comparable content from different instruments. Items showed little differential item functioning across age, diagnosis, or severity groups, and 92\% of the participants fit the model. CONCLUSIONS: ADL and IADL items from existing rehabilitation outcomes instruments that depend on skilled upper limb and hand use can be located along a single continuum, along with the new personal care and instrumental items of the AM-PAC addressing gaps in content. Results support the validity of the proposed definition of the Personal Care and Instrumental Activities dimension of function as a guide for future development of rehabilitation outcome instruments, such as linked, setting-specific short forms and computerized adaptive testing approaches.}, keywords = {*Self Efficacy, *Sickness Impact Profile, Activities of Daily Living/*classification/psychology, Adult, Aged, Aged, 80 and over, Disability Evaluation, Factor Analysis, Statistical, Female, Humans, Male, Middle Aged, Outcome Assessment (Health Care)/*methods/statistics \& numerical data, Questionnaires/*standards, Recovery of Function/physiology, Rehabilitation/*standards/statistics \& numerical data, Reproducibility of Results, Research Support, U.S. Gov{\textquoteright}t, Non-P.H.S., Research Support, U.S. Gov{\textquoteright}t, P.H.S., Sensitivity and Specificity}, author = {Coster, W. J. and Haley, S. M. and Andres, P. L. and Ludlow, L. H. and Bond, T. L. and Ni, P. S.} } @conference {943, title = {Recalibration of IRT item parameters in CAT: Sparse data matrices and missing data treatments}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {2003}, note = {(PDF file, 626 K}, address = {Chicago IL}, author = {Harmes, J. C. and Parshall, C. G. and Kromrey, J. D.} } @article {68, title = {The relationship between item exposure and test overlap in computerized adaptive testing}, journal = {Journal of Educational Measurement}, volume = {40}, number = {2}, year = {2003}, pages = {129-145}, abstract = {The purpose of this article is to present an analytical derivation for the mathematical form of an average between-test overlap index as a function of the item exposure index, for fixed-length computerized adaptive tests (CATs). This algebraic relationship is used to investigate the simultaneous control of item exposure at both the item and test levels. The results indicate that, in fixed-length CATs, control of the average between-test overlap is achieved via the mean and variance of the item exposure rates of the items that constitute the CAT item pool. The mean of the item exposure rates is easily manipulated. Control over the variance of the item exposure rates can be achieved via the maximum item exposure rate (r-sub(max)). Therefore, item exposure control methods which implement a specification of r-sub(max) (e.g., J. B. Sympson and R. D. Hetter, 1985) provide the most direct control at both the item and test levels. (PsycINFO Database Record (c) 2005 APA )}, keywords = {(Statistical), Adaptive Testing, Computer Assisted Testing, Human Computer, Interaction computerized adaptive testing, Item Analysis, Item Analysis (Test), Test Items}, author = {Chen, S-Y. and Ankemann, R. D. and Spray, J. A.} } @article {499, title = {The relationship between item exposure and test overlap in computerized adaptive testing}, journal = {Journal of Educational Measurement}, volume = {40}, year = {2003}, pages = {129-145}, author = {Chen, S. and Ankenmann, R. D. and Spray, J. A.} } @article {503, title = {The relationship between item exposure and test overlap in computerized adaptive testing}, journal = {Journal of Educational Measurement}, volume = {40}, year = {2003}, pages = {129-145}, author = {Chen, S. and Ankenmann, R. D. and Spray, J. A.} } @conference {1079, title = {A {\textquotedblleft}rearrangement procedure{\textquotedblright} for administering adaptive tests with review options}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {2002}, note = {{PDF file, 410 KB}}, address = {New Orleans LA}, author = {Papanastasiou, E. C.} } @conference {878, title = {Redeveloping the exposure control parameters of CAT items when a pool is modified}, booktitle = {Paper presented at the annual meeting of the American Educational Research Association}, year = {2002}, note = {{PDF file, 1.113 MB}}, address = {New Orleans LA}, author = {Chang, S-W. and Harris, D. J.} } @conference {1238, title = {Relative precision of ability estimation in polytomous CAT: A comparison under the generalized partial credit model and graded response model}, booktitle = {Paper presented at the annual meeting of the American Educational Research Association}, year = {2002}, note = {$\#$WA02-01 {PDF file, 735 KB}}, address = {New Orleans LA}, author = {Wang, S and Wang, T.} } @conference {958, title = {Reliability and decision accuracy of linear parallel form and multi stage tests with realistic and ideal item pools}, booktitle = {Paper presented at the International Conference on Computer-Based Testing and the Internet}, year = {2002}, address = {Winchester, England}, author = {Jodoin, M. G.} } @conference {1292, title = {The robustness of the unidimensional 3PL IRT model when applied to two-dimensional data in computerized adaptive testing}, booktitle = {Paper presented at the annual meeting of the American Educational Research Association}, year = {2002}, note = {{PDF file, 1.356 MB}}, address = {New Orleans LA}, author = {Zhao, J. C. and McMorris, R. F. and Pruzek, R. M. and Chen, R.} } @book {1689, title = {A rearrangement procedure for administering adaptive tests when review options are permitted}, year = {2001}, address = {Unpublished doctoral dissertation, Michigan State University}, author = {Papanastasiou, E. C.} } @conference {1251, title = {Refining a system for computerized adaptive testing pool creation}, booktitle = {Paper presented at the annual meeting of the American Educational Research Association}, year = {2001}, address = {Seattle WA}, author = {Way, W. D. and Swanson, l, and Stocking, M.} } @booklet {1565, title = {Refining a system for computerized adaptive testing pool creation (Research Report 01-18)}, year = {2001}, address = {Princeton NJ: Educational Testing Service}, author = {Way, W. D. and Swanson, L. and Steffen, M. and Stocking, M. L.} } @article {297, title = {Requerimientos, aplicaciones e investigaci{\'o}n en tests adaptativos informatizados [Requirements, applications, and investigation in computerized adaptive testing]}, journal = {Apuntes de Psicologia}, volume = {19}, number = {1}, year = {2001}, pages = {11-28}, abstract = {Summarizes the main requirements and applications of computerized adaptive testing (CAT) with emphasis on the differences between CAT and conventional computerized tests. Psychometric properties of estimations based on CAT, item selection strategies, and implementation software are described. Results of CAT studies in Spanish-speaking samples are described. Implications for developing a CAT measuring the English vocabulary of Spanish-speaking students are discussed. (PsycINFO Database Record (c) 2005 APA )}, keywords = {Computer Assisted Testing, English as Second Language, Psychometrics computerized adaptive testing}, author = {Olea D{\'\i}az, J. and Ponsoda Gil, V. and Revuelta Men{\'e}ndez, J. and Hontangas Beltr{\'a}n, P. and Abad, F. J.} } @article {548, title = {A real data simulation of computerized adaptive administration of the MMPI-A}, journal = {Computers in Human Behavior}, volume = {16}, year = {2000}, pages = {83-96}, author = {Forbey, J. D. and Handel, R. W. and Ben-Porath, Y. S.} } @article {133, title = {A real data simulation of computerized adaptive administration of the MMPI-A}, journal = {Computers in Human Behavior}, volume = {16}, number = {1}, year = {2000}, pages = {83-96}, abstract = {A real data simulation of computerized adaptive administration of the Minnesota Multiphasic Inventory-Adolescent (MMPI-A) was conducted using item responses from three groups of participants. The first group included 196 adolescents (age range 14-18) tested at a midwestern residential treatment facility for adolescents. The second group was the normative sample used in the standardization of the MMPI-A (Butcher, Williams, Graham, Archer, Tellegen, Ben-Porath, \& Kaemmer, 1992. Minnesota Multiphasic Inventory-Adolescent (MMPI-A): manual for administration, scoring, and interpretation. Minneapolis: University of Minnesota Press.). The third group was the clinical sample: used in the validation of the MMPI-A (Williams \& Butcher, 1989. An MMPI study of adolescents: I. Empirical validation of the study{\textquoteright}s scales. Personality assessment, 1, 251-259.). The MMPI-A data for each group of participants were run through a modified version of the MMPI-2 adaptive testing computer program (Roper, Ben-Porath \& Butcher, 1995. Comparability and validity of computerized adaptive testing with the MMPI-2. Journal of Personality Assessment, 65, 358-371.). To determine the optimal amount of item savings, each group{\textquoteright}s MMPI-A item responses were used to simulate three different orderings of the items: (1) from least to most frequently endorsed in the keyed direction; (2) from least to most frequently endorsed in the keyed direction with the first 120 items rearranged into their booklet order; and (3) all items in booklet order. The mean number of items administered for each group was computed for both classification and full- scale elevations for T-score cut-off values of 60 and 65. Substantial item administration savings were achieved for all three groups, and the mean number of items saved ranged from 50 items (10.7\% of the administered items) to 123 items (26.4\% of the administered items), depending upon the T-score cut-off, classification method (i.e. classification only or full-scale elevation), and group. (C) 2000 Elsevier Science Ltd. All rights reserved.}, author = {Fobey, J. D. and Handel, R. W. and Ben-Porath, Y. S.} } @article {777, title = {Rescuing computerized testing by breaking Zipf{\textquoteright}s law}, journal = {Journal of Educational and Behavioral Statistics}, volume = {25}, year = {2000}, pages = {203-224}, author = {Wainer, H.,} } @article {791, title = {Response to Hays et al and McHorney and Cohen: Practical implications of item response theory and computerized adaptive testing: A brief summary of ongoing studies of widely used headache impact scales}, journal = {Medical Care}, volume = {38}, year = {2000}, pages = {73-82}, author = {Ware, J. E., Jr. and Bjorner, J. B. and Kosinski, M.} } @article {356, title = {A review of CAT review}, journal = {Popular Measurement}, volume = {3}, number = {1}, year = {2000}, pages = {47-49}, abstract = {Studied the effects of answer review on results of a computerized adaptive test, the laboratory professional examination of the American Society of Clinical Pathologists. Results from 29,293 candidates show that candidates who changed answers were more likely to improve their scores. (SLD)}, author = {Sekula-Wacura, R.} } @conference {1267, title = {The rationale and principles of stratum scoring}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1999}, address = {Montreal, Canada}, author = {Wise, S. L.} } @article {786, title = {Reducing bias in CAT trait estimation: A comparison of approaches}, journal = {Applied Psychological Measurement}, volume = {23}, year = {1999}, pages = {263-278}, author = {Wang, T. and Hanson, B. H. and C.-M. H. Lau} } @conference {949, title = {Reducing item exposure without reducing precision (much) in computerized adaptive testing}, booktitle = { Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1999}, address = {Montreal, CA}, author = {Holmes, R. M. and Segall, D. O.} } @inbook {1758, title = {Research and development of a computer-adaptive test of listening comprehension in the less-commonly taught language Hausa}, year = {1999}, address = {M. Chalhoub-Deville (Ed.). Issues in computer-adaptive testing of reading proficiency. Cambridge, UK : Cambridge University Press.}, author = {Dunkel, P.} } @conference {1144, title = {Response time feedback on computer-administered tests}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1999}, address = {Montreal, Canada}, author = {Scrams, D. J. and Schnipke, D. L.} } @book {1726, title = {The robustness of the unidimensional 3PL IRT model when applied to two-dimensional data in computerized adaptive testing}, year = {1999}, note = {$\#$ZH99-1}, address = {Unpublished Ph.D. dissertation, State University of New York at Albany}, author = {Zhao, J. C.} } @booklet {1514, title = {The relationship between computer familiarity and performance on computer-based TOEFL test tasks (Research Report 98-08)}, year = {1998}, address = {Princeton NJ: Educational Testing Service}, author = {Taylor, C. and Jamieson, J. and Eignor, D. R. and Kirsch, I.} } @article {759, title = {Reviewing and changing answers on computer-adaptive and self-adaptive vocabulary tests}, journal = {Journal of Educational Measurement}, volume = {35}, year = {1998}, note = {(Also presented at National Council on Measurement in Education, 1996)}, pages = {328-345}, author = {Vispoel, W. P.} } @conference {891, title = {Realistic simulation procedures for item response data}, booktitle = {In T. Miller (Chair), High-dimensional simulation of item response data for CAT research. Psychometric Society}, year = {1997}, note = {Symposium presented at the annual meeting of the Psychometric Society, Gatlinburg TN.}, address = {Gatlinburg TN}, author = {Davey, T. and Nering, M. and Thompson, T.} } @conference {1186, title = {Relationship of response latency to test design, examinee ability, and item difficulty in computer-based test administration}, booktitle = {Paper presented at the annual meeting of the National Council on Measurement in Education}, year = {1997}, address = {Chicago IL}, author = {Swanson, D. B. and Featherman, C. M. and Case, A. M. and Luecht, RM and Nungester, R.} } @inbook {1860, title = {Reliability and construct validity of CAT-ASVAB}, year = {1997}, address = {W. A. Sands, B. K. Waters, and J. R. McBride (Eds.). Computerized adaptive testing: From inquiry to operation (pp. 169-179). Washington DC: American Psychological Association.}, author = {Moreno, K. E. and Segall, O. D.} } @inbook {270, title = {Research antecedents of applied adaptive testing}, booktitle = {Computerized adaptive testing: From inquiry to practice}, year = {1997}, pages = {47-57}, publisher = {American Psychological Association}, organization = {American Psychological Association}, edition = {xviii}, address = {Washington D.C. USA}, abstract = {(from the chapter) This chapter sets the stage for the entire computerized adaptive testing Armed Services Vocational Aptitude Battery (CAT-ASVAB) development program by describing the state of the art immediately preceding its inception. By the mid-l970s, a great deal of research had been conducted that provided the technical underpinnings needed to develop adaptive tests, but little research had been done to corroborate empirically the promising results of theoretical analyses and computer simulation studies. In this chapter, the author summarizes much of the important theoretical and simulation research prior to 1977. In doing so, he describes a variety of approaches to adaptive testing, and shows that while many methods for adaptive testing had been proposed, few practical attempts had been made to implement it. Furthermore, the few instances of adaptive testing were based primarily on traditional test theory, and were developed in laboratory settings for purposes of basic research. The most promising approaches, those based on item response theory and evaluated analytically or by means of computer simulations, remained to be proven in the crucible of live testing. (PsycINFO Database Record (c) 2004 APA, all rights reserved).}, keywords = {computerized adaptive testing}, author = {J. R. McBride}, editor = {B. K. Waters and J. R. McBride} } @article {371, title = {Revising item responses in computerized adaptive tests: A comparison of three models}, journal = {Applied Psychological Measurement}, volume = {21}, number = {2}, year = {1997}, note = {Sage Publications, US}, pages = {129-142}, abstract = {Interest in the application of large-scale computerized adaptive testing has focused attention on issues that arise when theoretical advances are made operational. One such issue is that of the order in which exaniinees address questions within a test or separately timed test section. In linear testing, this order is entirely under the control of the examinee, who can look ahead at questions and return and revise answers to questions. Using simulation, this study investigated three models that permit restricted examinee control over revising previous answers in the context of adaptive testing. Even under a worstcase model of examinee revision behavior, two of the models of permitting item revisions worked well in preserving test fairness and accuracy. One model studied may also preserve some cognitive processing styles developed by examinees for a linear testing environment. }, keywords = {computerized adaptive testing}, author = {Stocking, M. L.} } @article {687, title = {The role of item feedback in self-adapted testing}, journal = {Educational and Psychological Measurement}, volume = {57}, year = {1997}, pages = {85-98}, author = {Roos, L. L. and Wise, S. L. and Plake, B. S.} } @booklet {1346, title = {Recursive maximum likelihood estimation, sequential design, and computerized adaptive testing}, year = {1996}, address = {Princeton NJ: Educational Testing Service}, author = {Chang, Hua-Hua and Ying, Z.} } @booklet {1615, title = {Revising item responses in computerized adaptive testing: A comparison of three models (RR-96-12)}, year = {1996}, note = {$\#$ST96-12 [See APM paper 1997).}, address = {Princeton NJ: Educational Testing Service}, author = {Stocking, M. L.} } @book {1680, title = {Robustness of a unidimensional computerized testing mastery procedure with multidimensional testing data}, year = {1996}, note = {{PDF file, 1.838 MB}}, address = {Unpublished doctoral dissertation, University of Iowa, Iowa City IA}, author = {Lau, CA} } @conference {1286, title = {Recursive maximum likelihood estimation, sequential designs, and computerized adaptive testing}, booktitle = {Paper presented at the Eleventh Workshop on Item Response Theory}, year = {1995}, address = {University of Twente, the Netherlands}, author = {Ying, Z. and Chang, Hua-Hua} } @article {438, title = {Review of the book Computerized Adaptive Testing: A Primer}, journal = {Psychometrika}, volume = {4?}, year = {1995}, pages = {615-620}, author = {Andrich, D.} } @article {812, title = {The relationship between examinee anxiety and preference for self-adapted testing}, journal = {Applied Measurement in Education }, volume = {7}, year = {1994}, pages = {81-91}, author = {Wise, S. L. and Roos, L. L. and Plake, B. S., and Nebelsick-Gullett, L. J.} } @inbook {266, title = {Reliability of alternate computer adaptive tests}, booktitle = {Objective measurement, theory into practice}, volume = {II}, year = {1994}, publisher = {Ablex}, organization = {Ablex}, address = {New Jersey}, author = {Lunz, M. E. and Bergstrom, Betty A. and Wright, B. D.} } @article {703, title = {On the reliability of testlet-based tests}, journal = {Journal of Educational Measurement}, volume = {28}, year = {1991}, pages = {237-247}, author = {Sireci, S. G. and Wainer, H., and Thissen, D.} } @inbook {1907, title = {Reliability and measurement precision}, year = {1990}, address = {H. Wainer, N. J. Dorans, R. Flaugher, B. F. Green, R. J. Mislevy, L. Steinberg, and D. Thissen (Eds.), Computerized adaptive testing: A primer (pp. 161-186). Hillsdale NJ: Erlbaum.}, author = {Thissen, D.} } @inbook {1961, title = {A research proposal for field testing CAT for nursing licensure examinations}, year = {1990}, address = {Delegate Assembly Book of Reports 1989. Chicago: National Council of State Boards of Nursing.}, author = {A Zara} } @article {22, title = {A real-data simulation of computerized adaptive administration of the MMPI}, journal = {Psychological Assessment}, volume = {1}, number = {1}, year = {1989}, note = {Article}, pages = {18-22}, abstract = {A real-data simulation of computerized adaptive administration of the MMPI was conducted with data obtained from two personnel-selection samples and two clinical samples. A modification of the countdown method was tested to determine the usefulness, in terms of item administration savings, of several different test administration procedures. Substantial item administration savings were achieved for all four samples, though the clinical samples required administration of more items to achieve accurate classification and/or full-scale scores than did the personnel-selection samples. The use of normative item endorsement frequencies was found to be as effective as sample-specific frequencies for the determination of item administration order. The role of computerized adaptive testing in the future of personality assessment is discussed., (C) 1989 by the American Psychological Association}, keywords = {computerized adaptive testing}, author = {Ben-Porath, Y. S. and Slutske, W. S. and Butcher, J. N.} } @inbook {1933, title = {A research proposal for field testing CAT for nursing licensure examinations}, year = {1989}, address = {Delegate Assembly Book of Reports 1989. Chicago: National Council of State Boards of Nursing, Inc.}, author = {A Zara} } @conference {99, title = {The Rasch model and missing data, with an emphasis on tailoring test items}, booktitle = {annual meeting of the American Educational Research Association}, year = {1988}, month = {April 5-9}, address = {New Orleans, LA. USA}, abstract = {Many applications of educational testing have a missing data aspect (MDA). This MDA is perhaps most pronounced in item banking, where each examinee responds to a different subtest of items from a large item pool and where both person and item parameter estimates are needed. The Rasch model is emphasized, and its non-parametric counterpart (the Mokken scale) is considered. The possibility of tailoring test items in combination with their estimation is discussed; however, most methods for the estimation of item parameters are inadequate under tailoring. Without special measures, only marginal maximum likelihood produces adequate item parameter estimates under item tailoring. Fischer{\textquoteright}s approximate minimum-chi-square method for estimation of item parameters for the Rasch model is discussed, which efficiently produces item parameters. (TJH)}, author = {de Gruijter, D. N. M.} } @article {557, title = {The Rasch model and multi-stage testing}, journal = {Journal of Educational and Behavioral Statistics}, volume = {13}, year = {1988}, pages = {45-52}, author = {Glas, C. A. W.} } @inbook {1815, title = {On a Rasch-model-based test for non-computerized adaptive testing}, year = {1988}, address = {Langeheine, R. and Rost, J. (Ed.), Latent trait and latent class models. New York: Plenum Press.}, author = {Kubinger, K. D.} } @conference {1164, title = {A real-data simulation of adaptive MMPI administration}, booktitle = {Paper presented at the 23rd Annual Symposium on recent developments in the use of the MMPI}, year = {1988}, address = {St. Petersburg FL}, author = {Slutske, W. S. and Ben-Porath, Y. S. and Butcher, J. N.} } @booklet {1578, title = {Refinement of the Computerized Adaptive Screening Test (CAST) (Final Report, Contract No MDA203 06-C-0373)}, year = {1988}, address = {Washington, DC: American Institutes for Research.}, author = {Wise, L. L. and McHenry, J.J. and Chia, W.J. and Szenas, P.L. and J. R. McBride} } @inbook {1955, title = {Reducing the predictability of adaptive item sequences}, year = {1985}, address = {Proceedings of the 27th Annual Conference of the Military Testing Association, San Diego, 43-48.}, author = {Wetzel, C. D. and J. R. McBride} } @article {285, title = {Relationship between corresponding Armed Services Vocational Aptitude Battery (ASVAB) and computerized adaptive testing (CAT) subtests}, journal = {Applied Psychological Measurement}, volume = {8}, number = {2}, year = {1984}, note = {Sage Publications, US}, pages = {155-163}, abstract = {Investigated the relationships between selected subtests from the Armed Services Vocational Aptitude Battery (ASVAB) and corresponding subtests administered as computerized adaptive tests (CATs), using 270 17-26 yr old Marine recruits as Ss. Ss were administered the ASVAB before enlisting and approximately 2 wks after entering active duty, and the CAT tests were administered to Ss approximately 24 hrs after arriving at the recruit depot. Results indicate that 3 adaptive subtests correlated as well with ASVAB as did the 2nd administration of the ASVAB, although CAT subtests contained only half the number of items. Factor analysis showed CAT subtests to load on the same factors as the corresponding ASVAB subtests, indicating that the same abilities were being measured. It is concluded that CAT can achieve the same measurement precision as a conventional test, with half the number of items. (16 ref) }, keywords = {computerized adaptive testing}, author = {Moreno, K. E. and Wetzel, C. D. and J. R. McBride and Weiss, D. J.} } @article {2013, title = {Relationship Between Corresponding Armed Services Vocational Aptitude Battery (ASVAB) and Computerized Adaptive Testing (CAT) Subtests}, journal = {Applied Psychological Measurement}, volume = {8}, year = {1984}, pages = {155-163}, author = {Moreno, K. E. and Wetzel, C. D. and J. R. McBride and Weiss, D. J.} } @booklet {1481, title = {Relationship between corresponding Armed Services Vocational Aptitude Battery (ASVAB) and computerized adaptive testing (CAT) subtests (TR 83-27)}, year = {1983}, address = {San Diego CA: Navy Personnel Research and Development Center}, author = {Moreno, K. E. and Wetzel, D. C. and J. R. McBride and Weiss, D. J.} } @inbook {1851, title = {Reliability and validity of adaptive ability tests in a military setting}, year = {1983}, address = {D. J. Weiss (Ed.), New horizons in testing: Latent trait test theory and computerized adaptive testing (pp. 224-236). New York: Academic Press.}, author = {J. R. McBride and Martin, J. T.} } @booklet {1469, title = {Reliability and validity of adaptive ability tests in a military recruit population (Research Report 83-1)}, year = {1983}, address = {Minneapolis: Department of Psychology, Psychometric Methods Program, Computerized Testing Laboratory}, author = {J. R. McBride and Martin, J. T. and Weiss, D. J.} } @booklet {1456, title = {Reliability and validity of adaptive vs. conventional tests in a military recruit population (Research Rep. No. 83-1).}, year = {1983}, note = {{PDF file, 2.787 MB}}, address = {Minneapolis: University of Minnesota, Department of Psychology, Psychometric Methods Program, Computerized Adaptive Testing Laboratory.}, author = {Martin, J. T. and J. R. McBride and Weiss, D. J.} } @inbook {1951, title = {Robustness of adaptive testing to multidimensionality}, year = {1982}, note = {42 MB}}, address = {D. J. Weiss (Ed.), Proceedings of the 1982 Item Response Theory and Computerized Adaptive Testing Conference. Minneapolis: University of Minnesota, Department of Psychology, Psychometric Methods Program. {PDF file, 1.}, author = {Weiss, D. J. and Suhadolnik, D.} } @book {1679, title = {The Rasch model in computerized personality testing}, year = {1979}, address = {Ph.D. dissertation, University of Missouri, Columbia, 1979}, author = {Kunce, C. S.} } @booklet {1528, title = {A rapid item search procedure for Bayesian adaptive testing (Research Report 77-4)}, year = {1977}, address = {Minneapolis: University of Minnesota, Department of Psychology, Psychometric Methods Program}, author = {Vale, C. D. and Weiss, D. J.} } @conference {2223, title = {Real-data simulation of a proposal for tailored teting}, booktitle = {Third International Conference on Educational Testing}, year = {1977}, month = {06/1977}, address = {Leyden, The Netherlands}, author = {Killcross, M. C.} } @inbook {1975, title = {Reduction of Test Bias by Adaptive Testing}, year = {1977}, address = {D. J. Weiss (Ed.), Proceedings of the 1977 Computerized Adaptive Testing Conference. Minneapolis MN: University of Minnesota, Department of Psychology, Psychometric Methods Program.}, author = {Pine, S. M.} } @inbook {1791, title = {Reflections on adaptive testing}, year = {1976}, note = {{PDF file, 464 KB}}, address = {C. K. Clark (Ed.), Proceedings of the First Conference on Computerized Adaptive Testing (pp. 90-94). Washington DC: U.S. Government Printing Office.}, author = {Hansen, D. N.} } @booklet {1460, title = {Research on adaptive testing 1973-1976: A review of the literature}, year = {1976}, address = {Unpublished manuscript, University of Minnesota}, author = {J. R. McBride} } @booklet {1419, title = {A review of research in tailored testing (Report APRE No}, year = {1976}, address = {9/76, Farnborough, Hants, U. K.: Ministry of Defence, Army Personnel Research Establishment.)}, author = {Killcross, M. C.} } @inbook {1853, title = {Recent and projected developments in ability testing by computer}, year = {1974}, address = {Earl Jones (Ed.), Symposium Proceedings: Occupational Research and the Navy{\textendash}Prospectus 1980 (TR-74-14). San Diego, CA: Navy Personnel Research and Development Center.}, author = {J. R. McBride and Weiss, D. J.} } @article {816, title = {Response-contingent testing}, journal = {Review of Educational Research}, volume = {43}, year = {1973}, pages = {529-544}, author = {Wood, R. L.} } @booklet {1392, title = {A review of testing and decision-making procedures (Technical Bulletin No. 15}, year = {1973}, address = {Iowa City IA: American College Testing Program.}, author = {Hambleton, R. K.} } @article {620, title = {Robbins-Monro procedures for tailored testing}, journal = {Educational and Psychological Measurement}, volume = {31}, year = {1971}, pages = {3-31}, author = {Lord, F. M.,} } @article {512, title = {Reproduction of total test score through the use of sequential programmed tests}, journal = {Journal of Educational Measurement}, volume = {5}, year = {1968}, pages = {183-187}, author = {Cleary, T. A. and Linn, R. L. and Rock, D. A.} }