@article{smith_leeman-munk_shelton_mott_wiebe_lester_2019, title={A Multimodal Assessment Framework for Integrating Student Writing and Drawing in Elementary Science Learning}, volume={12}, ISSN={1939-1382 2372-0050}, url={http://dx.doi.org/10.1109/TLT.2018.2799871}, DOI={10.1109/TLT.2018.2799871}, abstractNote={Science learning is inherently multimodal, with students utilizing both drawings and writings to explain observations of physical phenomena. As such assessments in science should accommodate the many ways students express their understanding, especially given evidence that understanding is distributed across both drawing and writing. In recent years advanced automated assessment techniques that evaluate expressive student artifacts have emerged. However, these techniques have largely operated individually, each considering only a single mode. We propose a framework for the multimodal automated assessment of students’ writing and drawing to leverage the synergies inherent across modalities and create a more complete and accurate picture of a student's knowledge. We introduce a multimodal assessment framework as well as two computational techniques for automatically analyzing student writings and drawings: a convolutional neural network-based model for assessing student writing, and a topology-based model for assessing student drawing. Evaluations with elementary students’ writings and drawings collected with a tablet-based digital science notebook demonstrate that 1) each of the framework's two modalities provide an independent and complementary measure of student science learning, and 2) the computational methods are capable of accurately assessing student work from both modalities and offer the potential for integration in technology-rich learning environments for real-time formative assessment.}, number={1}, journal={IEEE Transactions on Learning Technologies}, publisher={Institute of Electrical and Electronics Engineers (IEEE)}, author={Smith, Andy and Leeman-Munk, Samuel and Shelton, Angi and Mott, Bradford and Wiebe, Eric and Lester, James}, year={2019}, month={Jan}, pages={3–15} }
 @article{leeman-munk_smith_mott_wiebe_lester_2015, title={Two Modes Are Better Than One: A Multimodal Assessment Framework Integrating Student Writing and Drawing}, volume={9112}, ISBN={["978-3-319-19772-2"]}, ISSN={["1611-3349"]}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-84948972893&partnerID=MN8TOARS}, DOI={10.1007/978-3-319-19773-9_21}, abstractNote={We are beginning to see the emergence of advanced automated assessment techniques that evaluate expressive student artifacts such as free-form written responses and sketches. These approaches have largely operated individually, each considering only a single mode. We hypothesize that there are synergies to be leveraged in multimodal assessments that can integrate multiple modalities of student responses to create a more complete and accurate picture of a student’s knowledge. In this paper, we introduce a novel multimodal assessment framework that integrates two techniques for automatically analyzing student artifacts: a deep learning-based model for assessing student writing, and a topology-based model for assessing student drawing. An evaluation of the framework with elementary students’ writing and drawing assessments demonstrate that 1) each of the framework’s two modalities provides an independent and complementary measure of student science learning, and 2) together, the multimodal framework significantly outperforms either uni-modal approach individually, demonstrating the potential synergistic benefits of multimodal assessment.}, journal={ARTIFICIAL INTELLIGENCE IN EDUCATION, AIED 2015}, author={Leeman-Munk, Samuel and Smith, Andy and Mott, Bradford and Wiebe, Eric and Lester, James}, year={2015}, pages={205–215} }