@article{vandenberg_mott_2023, title={"AI teaches itself": Exploring Young Learners' Perspectives on Artificial Intelligence for Instrument Development}, DOI={10.1145/3587102.3588778}, abstractNote={Children encounter and use artificial intelligence (AI) with regularity, but the depth of their understanding of AI is often limited. In service of growing an AI and technology-literate K-12 population, it is important for young learners to engage in AI learning activities early and often. To foster the design of AI curricula, it is essential to understand what young children already know and how they feel about AI. The nascent field of AI-related self-report instrument development focuses largely on adult populations or AI's use in specific contexts, such as medicine. There remains a critical need to develop an AI attitudinal survey for young learners (ages 9 to 11). Building upon the extant survey development work of those in education and AI, we have designed a brief survey on students' self-efficacy for AI, interest and motivation toward AI, and attitudes toward AI. We used cognitive interviewing processes to ensure the items in the survey were readable and understandable by young students. Preliminary findings indicate young students have mixed understanding of what AI is, what it can do, and how they feel about AI. We discuss implications for researchers and practitioners and provide an overview of our continuing efforts to validate this instrument.}, journal={PROCEEDINGS OF THE 2023 CONFERENCE ON INNOVATION AND TECHNOLOGY IN COMPUTER SCIENCE EDUCATION, ITICSE 2023, VOL 1}, author={Vandenberg, Jessica and Mott, Bradford}, year={2023}, pages={485–490} }
 @article{bae_feng_glazewski_hmelo-silver_chen_mott_lee_lester_2023, title={Co-designing a Classroom Orchestration Assistant for Game-based PBL Environments}, ISSN={["1559-7075"]}, DOI={10.1007/s11528-023-00903-4}, journal={TECHTRENDS}, author={Bae, Haesol and Feng, Chen and Glazewski, Krista and Hmelo-Silver, Cindy E. and Chen, Yuxin and Mott, Bradford W. and Lee, Seung Y. and Lester, James C.}, year={2023}, month={Nov} }
 @article{mott_gupta_glazewskianne_ottenbreit-leftwich_hmelo-silver_scribner_lee_lester_2023, title={Fostering Upper Elementary AI Education: Iteratively Refining a Use-Modify-Create Scaffolding Progression for AI Planning}, DOI={10.1145/3587103.3594170}, abstractNote={The growing ubiquity of artificial intelligence (AI) is reshaping much of daily life. This in turn is raising awareness of the need to introduce AI education throughout the K-12 curriculum so that students can better understand and utilize AI. A particularly promising approach for engaging young learners in AI education is game-based learning. In this work, we present our efforts to embed a unit on AI planning within an immersive game-based learning environment for upper elementary students (ages 8 to 11) that utilizes a scaffolding progression based on the Use-Modify-Create framework. Further, we present how the scaffolding progression is being refined based on findings from piloting the game with students.}, journal={PROCEEDINGS OF THE 2023 CONFERENCE ON INNOVATION AND TECHNOLOGY IN COMPUTER SCIENCE EDUCATION, ITICSE 2023, VOL. 2}, author={Mott, Bradford and Gupta, Anisha and GlazewskiAnne, Krista and Ottenbreit-Leftwich, Anne and Hmelo-Silver, Cindy and Scribner, Adam and Lee, Seung and Lester, James}, year={2023}, pages={647–647} }
 @article{vandenberg_min_catete_boulden_mott_2023, title={Leveraging Game Design Activities for Middle Grades AI Education in Rural Communities}, url={https://doi.org/10.1145/3582437.3587193}, DOI={10.1145/3582437.3587193}, abstractNote={The ever pervasive nature of artificial intelligence (AI) in our world necessitates a focus on fostering an AI literate society. Young children, those aged 11 to 14, are at a critical point in developing their dispositions toward and perceptions of science, technology, engineering, and mathematics (STEM), which influences their future education and career interests. Youth in rural areas are in particular need of access to AI learning opportunities to prepare them for the future workforce; digital games may be one way to attract young, rural students to STEM education and careers. In this paper, we explore how to introduce rural middle grades students to foundational AI concepts through digital game design activities. To inform our efforts and to establish an understanding of what these student populations as well as their teachers know about AI and games, we conducted a set of interviews and focus groups. In brief, students’ awareness and understanding of AI varied significantly, whereas teachers had limited knowledge of AI. Moreover, students shared great interest in playing and designing games. In support of our findings, we are developing a set of game design activities around five core AI concepts and ensuring the activities are of interest to our rural students.}, journal={PROCEEDINGS OF THE 18TH INTERNATIONAL CONFERENCE ON THE FOUNDATIONS OF DIGITAL GAMES, FDG 2023}, author={Vandenberg, Jessica and Min, Wookhee and Catete, Veronica and Boulden, Danielle and Mott, Bradford}, year={2023} }
 @article{monahan_vandenberg_gupta_smith_elsayed_fox_cheuoua_ringstaff_minogue_oliver_et al._2023, title={Multimodal CS Education Using a Scaffolded CSCL Environment}, url={https://doi.org/10.1145/3587103.3594181}, DOI={10.1145/3587103.3594181}, abstractNote={There is a growing need for 21st-century workers to be digitally literate and to possess computational thinking and collaborative problem-solving skills. Computer-supported collaborative learning (CSCL) focused on computational thinking can guide students toward the co-development of these skills. In this work, we present our approach to integrating virtual and physical learning modalities into InfuseCS, a CSCL environment. InfuseCS uses problem-based learning scenarios to situate upper elementary school students (ages 8 to 11) in a CSCL setting to foster their computational thinking and science knowledge construction as they collaborate to create digital narratives.}, journal={PROCEEDINGS OF THE 2023 CONFERENCE ON INNOVATION AND TECHNOLOGY IN COMPUTER SCIENCE EDUCATION, ITICSE 2023, VOL. 2}, author={Monahan, Robert and Vandenberg, Jessica and Gupta, Anisha and Smith, Andy and Elsayed, Rasha and Fox, Kimkinyona and Cheuoua, Aleata Hubbard and Ringstaff, Cathy and Minogue, James and Oliver, Kevin and et al.}, year={2023}, pages={645–645} }
 @article{vandenberg_min_gupta_catete_boulden_mott_2023, title={Toward AI-infused Game Design Activities for Rural Middle Grades Students}, url={https://doi.org/10.1145/3587103.3594199}, DOI={10.1145/3587103.3594199}, abstractNote={The ubiquity of artificial intelligence (AI) in everyday life suggests the need to ensure young students know about AI, its uses and limitations, and its benefits and risks, while enabling them to develop expertise in using AI-driven technologies. To support rural middle grades students and educators in learning and teaching AI concepts, we are designing AI-focused learning activities centered around the creation of digital gameplay experiences. To inform our designs, we conducted educator interviews and student focus groups to gain insights into their understanding of AI, their computer science background, and their knowledge and interest in gaming. Building on findings from these interviews and focus groups, we have designed a set of hands-on activities to elicit deeper feedback from students and educators on their preferences, points of confusion, and interests. In this work, we present our initial AI-infused game design activities.}, journal={PROCEEDINGS OF THE 2023 CONFERENCE ON INNOVATION AND TECHNOLOGY IN COMPUTER SCIENCE EDUCATION, ITICSE 2023, VOL. 2}, author={Vandenberg, Jessica and Min, Wookhee and Gupta, Anisha and Catete, Veronica and Boulden, Danielle and Mott, Bradford}, year={2023}, pages={644–644} }
 @article{saleh_phillips_hmelo-silver_glazewski_mott_lester_2022, title={A learning analytics approach towards understanding collaborative inquiry in a problem-based learning environment}, ISSN={["1467-8535"]}, DOI={10.1111/bjet.13198}, abstractNote={This exploratory paper highlights how problem-based learning (PBL) provided the pedagogical framework used to design and interpret learning analytics from C rystal Island: EcoJourneys , a collaborative game-based learning environment centred on supporting science inquiry. In C rystal Island: EcoJourneys , students work in teams of four, investigate the problem individually and then utilize a brainstorming board, an in-game PBL whiteboard that structured the collaborative inquiry process. The paper addresses a central question: how can PBL support the interpretation of the observed patterns in individual actions and collaborative interactions}, journal={BRITISH JOURNAL OF EDUCATIONAL TECHNOLOGY}, author={Saleh, Asmalina and Phillips, Tanner M. and Hmelo-Silver, Cindy E. and Glazewski, Krista D. and Mott, Bradford W. and Lester, James C.}, year={2022}, month={Feb} }
 @article{zhang_hutt_ocumpaugh_henderson_goslen_rowe_boyer_wiebe_mott_lester_2022, title={Investigating Student Interest and Engagement in Game-Based Learning Environments}, volume={13355}, ISBN={["978-3-031-11643-8"]}, ISSN={["1611-3349"]}, DOI={10.1007/978-3-031-11644-5_72}, abstractNote={As a cognitive and affective state, interest promotes engagement, facilitates self-regulated learning, and is positively associated with learning outcomes. Research has shown that interest interacts with prior knowledge, but few studies have investigated these issues in the context of adaptive game-based learning environments. Using three subscales from the User Engagement Scale, we examine data from middle school students (N = 77) who interacted with Crystal Island in their regular science class to explore the relationship between interest, knowledge, and learning. We found that interest is significantly related to performance (both knowledge assessment and game completion), suggesting that students with high interest are likely to perform better academically, but also be more engaged in the in-game objectives. These findings have implications both for designers who seek to identify students with lower interest and for those who hope to create adaptive supports.}, journal={ARTIFICIAL INTELLIGENCE IN EDUCATION, PT I}, author={Zhang, Jiayi and Hutt, Stephen and Ocumpaugh, Jaclyn and Henderson, Nathan and Goslen, Alex and Rowe, Jonathan P. and Boyer, Kristy Elizabeth and Wiebe, Eric and Mott, Bradford and Lester, James}, year={2022}, pages={711–716} }
 @article{park_mott_lee_gupta_jantaraweragul_glazewski_scribner_ottenbreit-leftwich_hmelo-silver_lester_2022, title={Investigating a visual interface for elementary students to formulate AI planning tasks}, volume={73}, ISSN={["2665-9182"]}, DOI={10.1016/j.cola.2022.101157}, abstractNote={Recent years have seen the rapid adoption of artificial intelligence (AI) in every facet of society. The ubiquity of AI has led to an increasing demand to integrate AI learning experiences into K-12 education. Early learning experiences incorporating AI concepts and practices are critical for students to better understand, evaluate, and utilize AI technologies. AI planning is an important class of AI technologies in which an AI-driven agent utilizes the structure of a problem to construct plans of actions to perform a task. Although a growing number of efforts have explored promoting AI education for K-12 learners, limited work has investigated effective and engaging approaches for delivering AI learning experiences to elementary students. In this article, we propose a visual interface to enable upper elementary students (grades 3-5, ages 8-11) to formulate AI planning tasks within a game-based learning environment. We present our approach to designing the visual interface as well as how the AI planning tasks are embedded within narrative-centered gameplay structured around a Use-Modify-Create scaffolding progression. Further, we present results from a study of upper elementary students using the visual interface. We discuss how the Use-Modify-Create approach supported student learning as well as discuss the misconceptions and usability issues students encountered while using the visual interface to formulate AI planning tasks.}, journal={JOURNAL OF COMPUTER LANGUAGES}, author={Park, Kyungjin and Mott, Bradford and Lee, Seung and Gupta, Anisha and Jantaraweragul, Katie and Glazewski, Krista and Scribner, J. Adam and Ottenbreit-Leftwich, Anne and Hmelo-Silver, Cindy E. and Lester, James}, year={2022}, month={Dec} }
 @article{ottenbreit-leftwich_glazewski_jeon_jantaraweragul_hmelo-silver_scribner_lee_mott_lester_2022, title={Lessons Learned for AI Education with Elementary Students and Teachers}, ISSN={["1560-4306"]}, DOI={10.1007/s40593-022-00304-3}, journal={INTERNATIONAL JOURNAL OF ARTIFICIAL INTELLIGENCE IN EDUCATION}, author={Ottenbreit-Leftwich, Anne and Glazewski, Krista and Jeon, Minji and Jantaraweragul, Katie and Hmelo-Silver, Cindy E. and Scribner, Adam and Lee, Seung and Mott, Bradford and Lester, James}, year={2022}, month={Sep} }
 @article{fahid_acosta_lee_carpenter_mott_bae_saleh_brush_glazewski_hmelo-silver_et al._2022, title={Multimodal Behavioral Disengagement Detection for Collaborative Game-Based Learning}, volume={13356}, ISBN={["978-3-031-11646-9"]}, ISSN={["1611-3349"]}, DOI={10.1007/978-3-031-11647-6_38}, abstractNote={Collaborative game-based learning environments offer significant promise for creating effective and engaging group learning experiences. These environments enable small groups of students to work together toward a common goal by sharing information, asking questions, and constructing explanations. However, students periodically disengage from the learning process, which negatively affects their learning, and the impacts are more severe in collaborative learning environments as disengagement can propagate, affecting participation across the group. Here, we introduce a multimodal behavioral disengagement detection framework that uses facial expression analysis in conjunction with natural language analyses of group chat. We evaluate the framework with students interacting with a collaborative game-based learning environment for middle school science education. The multimodal behavioral disengagement detection framework integrating both facial expression and group chat modalities achieves higher levels of predictive accuracy than those of baseline unimodal models.}, journal={ARTIFICIAL INTELLIGENCE IN EDUCATION: POSTERS AND LATE BREAKING RESULTS, WORKSHOPS AND TUTORIALS, INDUSTRY AND INNOVATION TRACKS, PRACTITIONERS AND DOCTORAL CONSORTIUM, PT II}, author={Fahid, Fahmid Morshed and Acosta, Halim and Lee, Seung and Carpenter, Dan and Mott, Bradford and Bae, Haesol and Saleh, Asmalina and Brush, Thomas and Glazewski, Krista and Hmelo-Silver, Cindy E. and et al.}, year={2022}, pages={218–221} }
 @article{spain_rowe_smith_goldberg_pokorny_mott_lester_2021, title={A reinforcement learning approach to adaptive remediation in online training}, volume={7}, ISSN={["1557-380X"]}, DOI={10.1177/15485129211028317}, abstractNote={Advances in artificial intelligence (AI) and machine learning can be leveraged to tailor training based on the goals, learning needs, and preferences of learners. A key component of adaptive training systems is tutorial planning, which controls how scaffolding is structured and delivered to learners to create dynamically personalized learning experiences. The goal of this study was to induce data-driven policies for tutorial planning using reinforcement learning (RL) to provide adaptive scaffolding based on the Interactive, Constructive, Active, Passive framework for cognitive engagement. We describe a dataset that was collected to induce RL-based scaffolding policies, and we present the results of our policy analyses. Results showed that the best performing policies optimized learning gains by inducing an adaptive fading approach in which learners received less cognitively engaging forms of remediation as they advanced through the training course. This policy was consistent with preliminary analyses that showed constructive remediation became less effective as learners progressed through the training session. Results also showed that learners’ prior knowledge impacted the type of scaffold that was recommended, thus showing evidence of an aptitude–treatment interaction. We conclude with a discussion of how AI-based training can be leveraged to enhance training effectiveness as well as directions for future research.}, journal={JOURNAL OF DEFENSE MODELING AND SIMULATION-APPLICATIONS METHODOLOGY TECHNOLOGY-JDMS}, author={Spain, Randall and Rowe, Jonathan and Smith, Andy and Goldberg, Benjamin and Pokorny, Robert and Mott, Bradford and Lester, James}, year={2021}, month={Jul} }
 @article{park_mott_lee_glazewski_scribner_ottenbreit-leftwich_hmelo-silver_lester_2021, title={Designing a Visual Interface for Elementary Students to Formulate AI Planning Tasks}, ISSN={["1943-6092"]}, DOI={10.1109/VL/HCC51201.2021.9576163}, abstractNote={Recent years have seen the rapid adoption of artificial intelligence (AI) in every facet of society. The ubiquity of AI has led to an increasing demand to integrate AI learning experiences into K-12 education. Early learning experiences incorporating AI concepts and practices are critical for students to better understand, evaluate, and utilize AI technologies. AI planning is an important class of AI technologies in which an AI-driven agent utilizes the structure of a problem to construct plans of actions to perform a task. Although a growing number of efforts have explored promoting AI education for K-12 learners, limited work has investigated effective and engaging approaches for delivering AI learning experiences to elementary students. In this paper, we propose a visual interface to enable upper elementary students (grades 3–5, ages 8–11) to formulate AI planning tasks within a game-based learning environment. We present our approach to designing the visual interface as well as how the AI planning tasks are embedded within narrative-centered gameplay structured around a Use-Modify-Create scaffolding progression. Further, we present results from a qualitative study of upper elementary students using the visual interface. We discuss how the Use-Modify-Create approach supported student learning as well as discuss the misconceptions and usability issues students encountered while using the visual interface to formulate AI planning tasks.}, journal={2021 IEEE SYMPOSIUM ON VISUAL LANGUAGES AND HUMAN-CENTRIC COMPUTING (VL/HCC 2021)}, author={Park, Kyungjin and Mott, Bradford and Lee, Seung and Glazewski, Krista and Scribner, J. Adam and Ottenbreit-Leftwich, Anne and Hmelo-Silver, Cindy E. and Lester, James}, year={2021} }
 @article{park_sohn_mott_min_saleh_glazewski_hmelo-silver_lester_2021, title={Detecting Disruptive Talk in Student Chat-Based Discussion within Collaborative Game-Based Learning Environments}, DOI={10.1145/3448139.3448178}, abstractNote={Collaborative game-based learning environments offer significant promise for creating engaging group learning experiences. Online chat plays a pivotal role in these environments by providing students with a means to freely communicate during problem solving. These chat-based discussions and negotiations support the coordination of students’ in-game learning activities. However, this freedom of expression comes with the possibility that some students might engage in undesirable communicative behavior. A key challenge posed by collaborative game-based learning environments is how to reliably detect disruptive talk that purposefully disrupt team dynamics and problem-solving interactions. Detecting disruptive talk during collaborative game-based learning is particularly important because if it is allowed to persist, it can generate frustration and significantly impede the learning process for students. This paper analyzes disruptive talk in a collaborative game-based learning environment for middle school science education to investigate how such behaviors influence students’ learning outcomes and varies across gender and students’ prior knowledge. We present a disruptive talk detection framework that automatically detects disruptive talk in chat-based group conversations. We further investigate both classic machine learning and deep learning models for the framework utilizing a range of dialogue representations as well as supplementary information such as student gender. Findings show that long short-term memory network (LSTM)-based disruptive talk detection models outperform competitive baseline models, indicating that the LSTM-based disruptive talk detection framework offers significant potential for supporting effective collaborative game-based learning through the identification of disruptive talk.}, journal={LAK21 CONFERENCE PROCEEDINGS: THE ELEVENTH INTERNATIONAL CONFERENCE ON LEARNING ANALYTICS & KNOWLEDGE}, author={Park, Kyungjin and Sohn, Hyunwoo and Mott, Bradford W. and Min, Wookhee and Saleh, Asmalina and Glazewski, Krista D. and Hmelo-Silver, Cindy E. and Lester, James C.}, year={2021}, pages={405–415} }
 @article{tian_wiggins_fahid_emerson_bounajim_smith_boyer_wiebe_mott_lester_2021, title={Modeling Frustration Trajectories and Problem-Solving Behaviors in Adaptive Learning Environments for Introductory Computer Science}, volume={12749}, ISBN={["978-3-030-78269-6"]}, ISSN={["1611-3349"]}, DOI={10.1007/978-3-030-78270-2_63}, abstractNote={Modeling a learner’s frustration in adaptive environments can inform scaffolding. While much work has explored momentary frustration, there is limited research investigating the dynamics of frustration over time and its relationship with problem-solving behaviors. In this paper, we clustered 86 undergraduate students into four frustration trajectories as they worked with an adaptive learning environment for introductory computer science. The results indicate that students who initially report high levels of frustration but then reported lower levels later in their problem solving were more likely to have sought help. These findings provide insight into how frustration trajectory models can guide adaptivity during extended problem-solving episodes.}, journal={ARTIFICIAL INTELLIGENCE IN EDUCATION (AIED 2021), PT II}, author={Tian, Xiaoyi and Wiggins, Joseph B. and Fahid, Fahmid Morshed and Emerson, Andrew and Bounajim, Dolly and Smith, Andy and Boyer, Kristy Elizabeth and Wiebe, Eric and Mott, Bradford and Lester, James}, year={2021}, pages={355–360} }
 @article{rachmatullah_reichsman_lord_dorsey_mott_lester_wiebe_2021, title={Modeling Secondary Students' Genetics Learning in a Game-Based Environment: Integrating the Expectancy-Value Theory of Achievement Motivation and Flow Theory}, volume={30}, ISSN={["1573-1839"]}, DOI={10.1007/s10956-020-09896-8}, number={4}, journal={JOURNAL OF SCIENCE EDUCATION AND TECHNOLOGY}, author={Rachmatullah, Arif and Reichsman, Frieda and Lord, Trudi and Dorsey, Chad and Mott, Bradford and Lester, James and Wiebe, Eric}, year={2021}, month={Aug}, pages={511–528} }
 @article{min_spain_saville_mott_brawner_johnston_lester_2021, title={Multidimensional Team Communication Modeling for Adaptive Team Training: A Hybrid Deep Learning and Graphical Modeling Framework}, volume={12748}, ISBN={["978-3-030-78291-7"]}, ISSN={["1611-3349"]}, DOI={10.1007/978-3-030-78292-4_24}, abstractNote={Team communication modeling offers great potential for adaptive learning environments for team training. However, the complex dynamics of team communication pose significant challenges for team communication modeling. To address these challenges, we present a hybrid framework integrating deep learning and probabilistic graphical models that analyzes team communication utterances with respect to the intent of the utterance and the directional flow of communication within the team. The hybrid framework utilizes conditional random fields (CRFs) that use deep learning-based contextual, distributed language representations extracted from team members’ utterances. An evaluation with communication data collected from six teams during a live training exercise indicate that linear-chain CRFs utilizing ELMo utterance embeddings (1) outperform both multi-task and single-task variants of stacked bidirectional long short-term memory networks using the same distributed representations of the utterances, (2) outperform a hybrid approach that uses non-contextual utterance representations for the dialogue classification tasks, and (3) demonstrate promising domain-transfer capabilities. The findings suggest that the hybrid multidimensional team communication analysis framework can accurately recognize speaker intent and model the directional flow of team communication to guide adaptivity in team training environments.}, journal={ARTIFICIAL INTELLIGENCE IN EDUCATION (AIED 2021), PT I}, author={Min, Wookhee and Spain, Randall and Saville, Jason D. and Mott, Bradford and Brawner, Keith and Johnston, Joan and Lester, James}, year={2021}, pages={293–305} }
 @article{saleh_chen_hmelo-silver_glazewski_mott_lester_2020, title={Coordinating scaffolds for collaborative inquiry in a game-based learning environment}, volume={57}, ISSN={["1098-2736"]}, DOI={10.1002/tea.21656}, abstractNote={Collaborative inquiry learning affords educators a context within which to support understanding of scientific practices, disciplinary core ideas, and crosscutting concepts. One approach to supporting collaborative science inquiry is through problem-based learning (PBL). However, there are two key challenges in scaffolding collaborative inquiry learning in technology rich environments. First, it is unclear how we might understand the impact of scaffolds that address multiple functions (e.g., to support inquiry and argumentation). Second, scaffolds take different forms, further complicating how to coordinate the forms and functions of scaffolds to support effective collaborative inquiry. To address these issues, we identify two functions that needed to be scaffolded, the PBL inquiry cycle and accountable talk. We then designed predefined hard scaffolds and just-in-time soft scaffolds that target the regulation of collaborative inquiry processes and accountable talk. Drawing on a mixed method approach, we examine how middle school students from a rural school engaged with Crystal Island: EcoJourneys for two weeks (N=45). Findings indicate that hard scaffolds targeting the PBL inquiry process}, number={9}, journal={JOURNAL OF RESEARCH IN SCIENCE TEACHING}, author={Saleh, Asmalina and Chen, Yuxin and Hmelo-Silver, Cindy E. and Glazewski, Krista D. and Mott, Bradford W. and Lester, James C.}, year={2020}, month={Nov}, pages={1490–1518} }
 @article{henderson_rowe_mott_brawner_baker_lester_2019, title={4D Affect Detection: Improving Frustration Detection in Game-Based Learning with Posture-Based Temporal Data Fusion}, volume={11625}, ISBN={["978-3-030-23203-0"]}, ISSN={["1611-3349"]}, DOI={10.1007/978-3-030-23204-7_13}, abstractNote={Recent years have seen growing interest in utilizing sensors to detect learner affect. Modeling frustration has particular significance because of its central role in learning. However, sensor-based affect detection poses important challenges. Motion-tracking cameras produce vast streams of spatial and temporal data, but relatively few systems have harnessed this data successfully to produce accurate run-time detectors of learner frustration outside of the laboratory. In this paper, we introduce a data-driven framework that leverages spatial and temporal posture data to detect learner frustration using deep neural network-based data fusion techniques. To train and validate the detectors, we utilize posture data collected with Microsoft Kinect sensors from students interacting with a game-based learning environment for emergency medical training. Ground-truth labels of learner frustration were obtained using the BROMP quantitative observation protocol. Results show that deep neural network-based late fusion techniques that combine spatial and temporal data yield significant improvements to frustration detection relative to baseline models.}, journal={ARTIFICIAL INTELLIGENCE IN EDUCATION (AIED 2019), PT I}, author={Henderson, Nathan L. and Rowe, Jonathan P. and Mott, Bradford W. and Brawner, Keith and Baker, Ryan and Lester, James C.}, year={2019}, pages={144–156} }
 @article{smith_leeman-munk_shelton_mott_wiebe_lester_2019, title={A Multimodal Assessment Framework for Integrating Student Writing and Drawing in Elementary Science Learning}, volume={12}, ISSN={1939-1382 2372-0050}, url={http://dx.doi.org/10.1109/TLT.2018.2799871}, DOI={10.1109/TLT.2018.2799871}, abstractNote={Science learning is inherently multimodal, with students utilizing both drawings and writings to explain observations of physical phenomena. As such assessments in science should accommodate the many ways students express their understanding, especially given evidence that understanding is distributed across both drawing and writing. In recent years advanced automated assessment techniques that evaluate expressive student artifacts have emerged. However, these techniques have largely operated individually, each considering only a single mode. We propose a framework for the multimodal automated assessment of students’ writing and drawing to leverage the synergies inherent across modalities and create a more complete and accurate picture of a student's knowledge. We introduce a multimodal assessment framework as well as two computational techniques for automatically analyzing student writings and drawings: a convolutional neural network-based model for assessing student writing, and a topology-based model for assessing student drawing. Evaluations with elementary students’ writings and drawings collected with a tablet-based digital science notebook demonstrate that 1) each of the framework's two modalities provide an independent and complementary measure of student science learning, and 2) the computational methods are capable of accurately assessing student work from both modalities and offer the potential for integration in technology-rich learning environments for real-time formative assessment.}, number={1}, journal={IEEE Transactions on Learning Technologies}, publisher={Institute of Electrical and Electronics Engineers (IEEE)}, author={Smith, Andy and Leeman-Munk, Samuel and Shelton, Angi and Mott, Bradford and Wiebe, Eric and Lester, James}, year={2019}, month={Jan}, pages={3–15} }
 @article{saleh_hmelo-silver_glazewski_mott_chen_rowe_lester_2019, title={Collaborative inquiry play A design case to frame integration of collaborative problem solving with story-centric games}, volume={120}, ISSN={["1758-6909"]}, DOI={10.1108/ILS-03-2019-0024}, abstractNote={Purpose This paper aims to present a model of collaborative inquiry play: rule-based imaginary situations that provide challenging problems and support agentic multiplayer interactions (c.f., Vygotsky, 1967; Salen and Zimmerman, 2003). Drawing on problem-based learning (PBL, Hmelo-Silver, 2004), this paper provides a design case to articulate the relationship between the design goals and the game-based learning environment. Design/methodology/approach Drawing on conjecture mapping (Sandoval, 2014), this paper presents an iterative development of the conjecture map for crystal island: ecojourneys and highlights the development of the story and tools in crystal island: ecojourneys , an immersive game based on PBL pedagogy. By articulating this development, the authors highlight the affordances and constraints of designing for collaborative inquiry play and address challenges in supporting learner agency. Findings The PBL inquiry process served as the foundation of collaborative inquiry play. Attending to the rules of inquiry fostered student agency, and in turn, playful engagement in the game-based learning environment. Agency however meant holding students accountable to actions undertaken, especially as it pertained to generating group-based explanations and reflecting on productive collaboration. Moreover, socially shared regulation of learning and systems thinking concepts (i.e. phenomenon, mechanisms, and components) must also be externalized in representations and interactions in the game such that students have the agency to decide on their learning paths. Originality/value This paper presents the model of collaborative inquiry play and highlights how to support player agency and design content-rich play environments which are not always completely open.}, number={9/10}, journal={INFORMATION AND LEARNING SCIENCES}, author={Saleh, Asmalina and Hmelo-Silver, Cindy E. and Glazewski, Krista D. and Mott, Bradford and Chen, Yuxin and Rowe, Jonathan P. and Lester, James C.}, year={2019}, month={Oct}, pages={547–566} }
 @article{geden_smith_campbell_spain_amos-binks_mott_feng_lester_2019, title={Construction and Validation of an Anticipatory Thinking Assessment}, volume={10}, ISSN={["1664-1078"]}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-85077306542&partnerID=MN8TOARS}, DOI={10.3389/fpsyg.2019.02749}, abstractNote={Anticipatory thinking is a critical cognitive skill for successfully navigating complex, ambiguous systems in which individuals must analyze system states, anticipate outcomes, and forecast future events. For example, in military planning, intelligence analysis, business, medicine, and social services, individuals must use information to identify warnings, anticipate a spectrum of possible outcomes, and forecast likely futures in order to avoid tactical and strategic surprise. Existing methods for examining anticipatory thinking skill have relied upon task-specific behavioral measures or are resource-intensive, both of which are challenging to scale. Given the increasing importance of anticipatory thinking in many domains, developing a generic assessment of this skill and identifying the underlying cognitive mechanisms supporting it are paramount. The work reported here focuses on the development and validation of the anticipatory thinking assessment (ANTA) for measuring the divergent generative process of anticipatory thinking. Two-hundred and ten participants completed the ANTA, which required them to anticipate possible risks, opportunities, trends, or other uncertainties associated with a focal topic. Responses to the anticipatory thinking and divergent thinking tasks were rated by trained raters on a five-point scale according to the uniqueness, specificity, and remoteness of responses. Results supported the ANTA’s construct validity, convergent validity, and discriminant validity. We also explored the relationship between the ANTA scores and certain psychological traits and cognitive measures (need for cognition, need for closure, and mindfulness). Our findings suggest that the ANTA is a psychometrically valid instrument that may help researchers investigate anticipatory thinking in new contexts.}, journal={FRONTIERS IN PSYCHOLOGY}, author={Geden, Michael and Smith, Andy and Campbell, James and Spain, Randall and Amos-Binks, Adam and Mott, Bradford W. and Feng, Jing and Lester, James}, year={2019}, month={Dec} }
 @article{mott_taylor_lee_rowe_saleh_glazewski_hmelo-silver_lester_2019, title={Designing and Developing Interactive Narratives for Collaborative Problem-Based Learning}, volume={11869}, ISBN={["978-3-030-33893-0"]}, ISSN={["1611-3349"]}, DOI={10.1007/978-3-030-33894-7_10}, abstractNote={Narrative and collaboration are two core features of rich interactive learning. Narrative-centered learning environments offer significant potential for supporting student learning. By contextualizing learning within interactive narratives, these environments leverage students’ innate facilities for developing understandings through stories. Computer-supported collaborative learning environments offer students rich, collaborative learning experiences in which small groups of students engage in constructing artifacts, addressing disciplinary challenges, and solving problems. Narrative and collaboration have distinct affordances for learning, but combining them poses significant challenges. In this paper, we present initial work on solving this problem by introducing collaborative narrative-centered learning environments. These environments will enable small groups of students to collaboratively solve problems in rich multi-participant storyworlds. We propose a novel framework for designing and developing these environments, which we are using to create a collaborative narrative-centered learning environment for middle school ecosystems education. In the learning environment, students work on problem-solving scenarios centered on how to support optimal fish health in aquatic environments. Results from pilot testing the learning environment with 45 students suggest it supports the creation of engaging and effective collaborative narrative-centered learning experiences.}, journal={INTERACTIVE STORYTELLING, ICIDS 2019}, author={Mott, Bradford W. and Taylor, Robert G. and Lee, Seung Y. and Rowe, Jonathan P. and Saleh, Asmalina and Glazewski, Krista D. and Hmelo-Silver, Cindy E. and Lester, James C.}, year={2019}, pages={86–100} }
 @article{ozer_penilla_spain_mott_woodson_lester_2019, title={HEALTH QUEST: PROMOTING ADOLESCENTS' HEALTH SCIENCE CAREER INTERESTS THROUGH TECHNOLOGY-RICH LEARNING EXPERIENCES}, volume={64}, ISSN={["1879-1972"]}, DOI={10.1016/j.jadohealth.2018.10.279}, abstractNote={The purpose of the Health Quest project is to create an intelligent game-based learning environment and associated resources to increase adolescents' knowledge of, interest in and self-efficacy to pursue health science careers. Science, technology, engineering and mathematics (STEM) fields are among the fastest growing career opportunities, yet women and ethnic minorities remain underrepresented in these fields. As adolescence offers a key window to promote interest in and increase self-efficacy to pursue health research careers and the health professions, Health Quest focuses on an adolescent student population. Health Quest will be pilot tested and used in classrooms across North Carolina and California, and will also be featured in informal learning settings including the North Carolina Museum of Natural Sciences and in after-school programs in San Francisco. To support the project’s first aim of designing and developing a series of Health Quest Career Adventure Game episodes and interactive video interviews to promote students’ interest in health science careers, the project team conducted an extensive review of recent NIH Science Education Partnership Award (SEPA) projects awarded over the past three years. The purpose of the review was to identify existing programs and online resources that have been developed to promote students’ interest in health science careers and to identify any unique resources or game-based learning environments that aim to promote self-efficacy or mastery experiences towards building competency and interest in health science career fields. We conducted a review of existing SEPA projects awarded from 2015 to 2018 to identify online resources and game-based learning experiences that have been designed to promote students' interest in health science careers. Projects were reviewed according to content focus, instructional approach and resources provided (e.g., websites, podcasts, videos and online-games). Forty five of the 48 SEPA projects reviewed had websites. Projects focused primarily on the life sciences, such as genomics. A majority of projects involved engaging students in the classroom, including providing opportunities in the lab, and used an experiential and mentor-based approach to provide students with unique hands-on learning opportunities to promote students’ knowledge and interest in STEM fields. Other projects involved teacher development. Game-based learning technologies offer significant potential for increasing students’ interest in health science careers. The results of our review showed that while SEPA programs focus on promoting interests and STEM opportunities by engaging students in hands-on classroom and laboratory-based activities, there is a lack of technology-based resources that can be used to extend outreach to students who would not otherwise have access to these programs. We hypothesize that by leveraging the intrinsic motivation of game-play, participant agency, and personalized learning, we can create engaging learning experiences that enable students to explore and gain confidence in pursuing health science careers.}, number={2}, journal={JOURNAL OF ADOLESCENT HEALTH}, author={Ozer, Elizabeth M. and Penilla, Carlos and Spain, Randall D. and Mott, Bradford W. and Woodson, Donald and Lester, James C.}, year={2019}, month={Feb}, pages={S134–S134} }
 @article{min_park_wiggins_mott_wiebe_boyer_lester_2019, title={Predicting Dialogue Breakdown in Conversational Pedagogical Agents with Multimodal LSTMs}, volume={11626}, ISBN={["978-3-030-23206-1"]}, ISSN={["1611-3349"]}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-85068335512&partnerID=MN8TOARS}, DOI={10.1007/978-3-030-23207-8_37}, abstractNote={Recent years have seen a growing interest in conversational pedagogical agents. However, creating robust dialogue managers for conversational pedagogical agents poses significant challenges. Agents’ misunderstandings and inappropriate responses may cause breakdowns in conversational flow, lead to breaches of trust in agent-student relationships, and negatively impact student learning. Dialogue breakdown detection (DBD) is the task of predicting whether an agent’s utterance will cause a breakdown in an ongoing conversation. A robust DBD framework can support enhanced user experiences by choosing more appropriate responses, while also offering a method to conduct error analyses and improve dialogue managers. This paper presents a multimodal deep learning-based DBD framework to predict breakdowns in student-agent conversations. We investigate this framework with dialogues between middle school students and a conversational pedagogical agent in a game-based learning environment. Results from a study with 92 middle school students demonstrate that multimodal long short-term memory network (LSTM)-based dialogue breakdown detectors incorporating eye gaze features achieve high predictive accuracies and recall rates, suggesting that multimodal detectors can play an important role in designing conversational pedagogical agents that effectively engage students in dialogue.}, journal={ARTIFICIAL INTELLIGENCE IN EDUCATION, AIED 2019, PT II}, author={Min, Wookhee and Park, Kyungjin and Wiggins, Joseph and Mott, Bradford and Wiebe, Eric and Boyer, Kristy Elizabeth and Lester, James}, year={2019}, pages={195–200} }
 @article{wiggins_kulkarni_min_boyer_mott_wiebe_lester_2019, title={Take the Initiative: Mixed Initiative Dialogue Policies for Pedagogical Agents in Game-Based Learning Environments}, volume={11626}, ISBN={["978-3-030-23206-1"]}, ISSN={["1611-3349"]}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-85068350756&partnerID=MN8TOARS}, DOI={10.1007/978-3-030-23207-8_58}, abstractNote={Pedagogical agents have been shown to be highly effective for supporting learning in a broad range of contexts, including game-based learning. However, there are key open questions around how to design dialogue policies for pedagogical agents that support students in game-based learning environments. This paper reports on a study to investigate two different agent dialogue policies with regard to conversational initiative, a core consideration in dialogue system design. In the User Initiative policy, only the student could initiate conversations with the agent, while in the Mixed Initiative policy, both the agent and the student could initiate conversations. In a study with 67 college students, results showed that the Mixed Initiative policy not only promoted more conversation, but also better supported the goals of the game-based learning environment by fostering exploration, yielding better performance on in-game assessments, and creating higher student engagement.}, journal={ARTIFICIAL INTELLIGENCE IN EDUCATION, AIED 2019, PT II}, author={Wiggins, Joseph B. and Kulkarni, Mayank and Min, Wookhee and Boyer, Kristy Elizabeth and Mott, Bradford and Wiebe, Eric and Lester, James}, year={2019}, pages={314–318} }
 @article{defalco_rowe_paquette_georgoulas-sherry_brawner_mott_baker_lester_2018, title={Detecting and Addressing Frustration in a Serious Game for Military Training}, volume={28}, ISSN={["1560-4306"]}, DOI={10.1007/s40593-017-0152-1}, abstractNote={Tutoring systems that are sensitive to affect show considerable promise for enhancing student learning experiences. Creating successful affective responses requires considerable effort both to detect student affect and to design appropriate responses to affect. Recent work has suggested that affect detection is more effective when both physical sensors and interaction logs are used, and that context-sensitive design of affective feedback is necessary to enhance engagement and improve learning. In this paper, we provide a comprehensive report on a multi-part study that integrates detection, validation, and intervention into a unified approach. This paper examines the creation of both sensor-based and interaction-based detectors of student affect, producing successful detectors of student affect. In addition, it reports results from an investigation of motivational feedback messages designed to address student frustration, and investigates whether linking these interventions to detectors improves outcomes. Our results are mixed, finding that self-efficacy enhancing interventions based on interaction-based affect detectors enhance outcomes in one of two experiments investigating affective interventions. This work is conducted in the context of the GIFT framework for intelligent tutoring, and the TC3Sim game-based simulation that provides training for first responder skills.}, number={2}, journal={INTERNATIONAL JOURNAL OF ARTIFICIAL INTELLIGENCE IN EDUCATION}, author={DeFalco, Jeanine A. and Rowe, Jonathan P. and Paquette, Luc and Georgoulas-Sherry, Vasiliki and Brawner, Keith and Mott, Bradford W. and Baker, Ryan S. and Lester, James C.}, year={2018}, month={Jun}, pages={152–193} }
 @article{catete_lytle_dong_boulden_akram_houchins_barnes_wiebe_lester_mott_et al._2018, title={Infusing Computational Thinking into Middle Grade Science Classrooms: Lessons Learned}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-85056713650&partnerID=MN8TOARS}, DOI={10.1145/3265757.3265778}, abstractNote={There is a growing need to present all students with an opportunity to learn computer science and computational thinking (CT) skills during their primary and secondary education. Traditionally, these opportunities are available outside of the core curriculum as stand-alone courses often taken by those with preparatory privilege. Researchers have identified the need to integrate CT into core classes to provide equitable access to these critical skills. We have worked in a research-practice partnership with two magnet middle schools focused on digital sciences to develop and implement computational thinking into life sciences classes. In this report, we present initial lessons learned while conducting our design-based implementation research on integrating computational thinking into middle school science classes. These case studies suggest that several factors including teacher engagement, teacher attitudes, student prior experience with CS/CT, and curriculum design can all impact student engagement in integrated science-CT lessons.}, journal={WIPSCE'18: PROCEEDINGS OF THE 13TH WORKSHOP IN PRIMARY AND SECONDARY COMPUTING EDUCATION}, publisher={ACM Press}, author={Catete, Veronica and Lytle, Nicholas and Dong, Yihuan and Boulden, Danielle and Akram, Bita and Houchins, Jennifer and Barnes, Tiffany and Wiebe, Eric and Lester, James and Mott, Bradford and et al.}, year={2018}, pages={109–114} }
 @inproceedings{buffum_ying_zheng_boyer_wiebe_mott_blackburn_lester_2018, title={Introducing the Computer Science Concept of Variables in Middle School Science Classrooms}, ISBN={9781450351034}, url={http://dx.doi.org/10.1145/3159450.3159545}, DOI={10.1145/3159450.3159545}, abstractNote={The K-12 Computer Science Framework has established that students should be learning about the computer science concept of variables as early as middle school, although the field has not yet determined how this and other related concepts should be introduced. Secondary school computer science curricula such as Exploring CS and AP CS Principles often teach the concept of variables in the context of algebra, which most students have already encountered in their mathematics courses. However, when strategizing how to introduce the concept at the middle school level, we confront the reality that many middle schoolers have not yet learned algebra. With that challenge in mind, this position paper makes a case for introducing the concept of variables in the context of middle school science. In addition to an analysis of existing curricula, the paper includes discussion of a day-long pilot study and the consequent teacher feedback that further supports the approach. The CS For All initiative has increased interest in bringing computer science to middle school classrooms; this paper makes an argument for doing so in a way that can benefit students' learning of both computer science and core science content.}, booktitle={Proceedings of the 49th ACM Technical Symposium on Computer Science Education - SIGCSE '18}, publisher={ACM Press}, author={Buffum, Philip Sheridan and Ying, Kimberly Michelle and Zheng, Xiaoxi and Boyer, Kristy Elizabeth and Wiebe, Eric N. and Mott, Bradford W. and Blackburn, David C. and Lester, James C.}, year={2018}, pages={906–911} }
 @article{geden_smith_campbell_amos-binks_mott_feng_lester_2018, title={Towards Adaptive Support for Anticipatory Thinking}, DOI={10.1145/3183654.3183665}, abstractNote={Adaptive training and support technologies have been used to improve training and performance in a number of domains. However, limited work on adaptive training has examined anticipatory thinking, which is the deliberate, divergent exploration and analysis of relevant futures to avoid surprise. Anticipatory thinking engages the process of imagining how uncertainties impact the future, helps identify leading indicators and causal dependencies of future scenarios, and complements forecasting, which focuses on assessing the likelihood of outcomes. It is particularly important for intelligence analysis, mission planning, and strategic forecasting, wherein practitioners apply prospective sense-making, scenario planning, and other methodologies to identify possible options and their effects during decision making processes. However, there is currently no underlying cognitive theory supporting specific anticipatory thinking methodologies, no adaptive technologies to support their training, and no existing measures to assess their efficacy. We are engaged in an ongoing effort to design adaptive technologies to support the acquisition and measurement of anticipatory thinking. As a first step toward adaptive environments that support the acquisition and application of anticipatory thinking competencies, we have developed a task to measure anticipatory thinking in which participants explore uncertainties and the impacts on the future given a particular topic. We present preliminary results from a study to examine the validity of this measure and discuss multiple factors that affect anticipatory thinking including attention, inhibitory control, need for cognition, need for closure, convergent thinking, and divergent thinking. We then introduce design principles for supporting training, application, and assessment of anticipatory thinking.}, journal={PROCEEDINGS OF THE TECHNOLOGY, MIND, AND SOCIETY CONFERENCE (TECHMINDSOCIETY'18)}, author={Geden, Michael and Smith, Andy and Campbell, James and Amos-Binks, Adam and Mott, Bradford and Feng, Jing and Lester, James}, year={2018} }
 @article{pezzullo_wiggins_frankosky_min_boyer_mott_wiebe_lester_2017, title={"Thanks Alisha, Keep in Touch": Gender Effects and Engagement with Virtual Learning Companions}, volume={10331}, ISBN={["978-3-319-61424-3"]}, ISSN={["1611-3349"]}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-85022211435&partnerID=MN8TOARS}, DOI={10.1007/978-3-319-61425-0_25}, abstractNote={Virtual learning companions have shown significant potential for supporting students. However, there appear to be gender differences in their effectiveness. In order to support all students well, it is important to develop a deeper understanding of the role that student gender plays during interactions with learning companions. This paper reports on a study to explore the impact of student gender and learning companion design. In a three-condition study, we examine middle school students’ interactions in a game-based learning environment that featured one of the following: (1) a learning companion deeply integrated into the narrative of the game; (2) a learning companion whose backstory and personality were not integrated into the narrative but who provided equivalent task support; and (3) no learning companion. The results show that girls were significantly more engaged than boys, particularly with the narrative-integrated agent, while boys reported higher mental demand with that agent. Even when controlling for video game experience and prior knowledge, the gender effects held. These findings contribute to the growing understanding that learning companions must adapt to students’ gender in order to facilitate the most effective learning interactions.}, journal={ARTIFICIAL INTELLIGENCE IN EDUCATION, AIED 2017}, author={Pezzullo, Lydia G. and Wiggins, Joseph B. and Frankosky, Megan H. and Min, Wookhee and Boyer, Kristy Elizabeth and Mott, Bradford W. and Wiebe, Eric N. and Lester, James C.}, year={2017}, pages={299–310} }
 @article{min_frankosky_mott_wiebe_boyer_lester_2017, title={Inducing Stealth Assessors from Game Interaction Data}, volume={10331}, ISBN={["978-3-319-61424-3"]}, ISSN={["1611-3349"]}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-85022230700&partnerID=MN8TOARS}, DOI={10.1007/978-3-319-61425-0_18}, abstractNote={A key untapped feature of game-based learning environments is their capacity to generate a rich stream of fine-grained learning interaction data. The learning behaviors captured in these data provide a wealth of information on student learning, which stealth assessment can utilize to unobtrusively draw inferences about student knowledge to provide tailored problem-solving support. In this paper, we present a long short-term memory network (LSTM)-based stealth assessment framework that takes as input an observed sequence of raw game-based learning environment interaction data along with external pre-learning measures to infer students’ post-competencies. The framework is evaluated using data collected from 191 middle school students interacting with a game-based learning environment for middle grade computational thinking. Results indicate that LSTM-based stealth assessors induced from student game-based learning interaction data outperform comparable models that required labor-intensive hand-engineering of input features. The findings suggest that the LSTM-based approach holds significant promise for evidence modeling in stealth assessment.}, journal={ARTIFICIAL INTELLIGENCE IN EDUCATION, AIED 2017}, author={Min, Wookhee and Frankosky, Megan H. and Mott, Bradford W. and Wiebe, Eric N. and Boyer, Kristy Elizabeth and Lester, James C.}, year={2017}, pages={212–223} }
 @article{rowe_lobene_mott_lester_2017, title={Play in the Museum: Design and Development of a Game-Based Learning Exhibit for Informal Science Education}, volume={9}, ISSN={["1942-3896"]}, DOI={10.4018/ijgcms.2017070104}, abstractNote={Digital games have been found to yield effective and engaging learning experiences across a broad range of subjects. Much of this research has been conducted in laboratory and K-12 classrooms. Recent advances in game technologies are expanding the range of educational contexts where game-based learning environments can be deployed, including informal settings such as museums and science centers. In this article, we describe the design, development, and formative evaluation of F UTURE W ORLDS , a prototype game-based exhibit for collaborative explorations of sustainability in science museums. We report findings from a museum pilot study that investigated the influence of visitors’ individual differences on learning and engagement. Results indicate that visitors showed significant gains in sustainability knowledge as well as high levels of engagement in a free-choice learning environment with F UTURE W ORLDS . These findings point toward the importance of designing game-based learning exhibits that address the distinctive design challenges presented by museum settings.}, number={3}, journal={INTERNATIONAL JOURNAL OF GAMING AND COMPUTER-MEDIATED SIMULATIONS}, author={Rowe, Jonathan P. and Lobene, Eleni V. and Mott, Bradford W. and Lester, James C.}, year={2017}, pages={96–113} }
 @article{buffum_frankosky_boyer_wiebe_mott_lester_2016, title={Collaboration and Gender Equity in Game-Based Learning for Middle School Computer Science}, volume={18}, ISSN={1521-9615}, url={http://dx.doi.org/10.1109/MCSE.2016.37}, DOI={10.1109/mcse.2016.37}, abstractNote={Game-based learning environments can deliver robust learning gains and have a significant capacity to engage students. Yet, they can unintentionally disadvantage students with less prior game experience. This article presents evidence that a collaborative gameplay approach can effectively address this problem at the middle school level. In an iterative, designed-based research study, the authors first used an experimental pilot study to investigate the nature of collaboration in the Engage game-based learning environment and then deployed Engage in a full classroom study to measure its effectiveness at supporting all students during computer science learning. In early phases of the intervention, male students outpaced female peers in learning gains. However, female students caught up during a multiweek classroom implementation. These findings provide evidence that gender differences could dissipate over time within collaborative game-based learning experiences in computer science.}, number={2}, journal={Computing in Science & Engineering}, publisher={Institute of Electrical and Electronics Engineers (IEEE)}, author={Buffum, Philip Sheridan and Frankosky, Megan and Boyer, Kristy Elizabeth and Wiebe, Eric N. and Mott, Bradford W. and Lester, James C.}, year={2016}, month={Mar}, pages={18–28} }
 @inbook{smith_aksit_min_wiebe_mott_lester_2016, place={Cham, Switzerland}, title={Integrating Real-Time Drawing and Writing Diagnostic Models: An Evidence-Centered Design Framework for Multimodal Science Assessment}, volume={9684}, ISBN={9783319395821 9783319395838}, ISSN={0302-9743 1611-3349}, url={http://dx.doi.org/10.1007/978-3-319-39583-8_16}, DOI={10.1007/978-3-319-39583-8_16}, abstractNote={Interactively modeling science phenomena enables students to develop rich conceptual understanding of science. While this understanding is often assessed through summative, multiple-choice instruments, science notebooks have been used extensively in elementary and secondary grades as a mechanism to promote and reveal reflection through both drawing and writing. Although each modality has been studied individually, obtaining a comprehensive view of a student’s conceptual understanding requires analyses of knowledge represented across both modalities. Evidence-centered design (ECD) provides a framework for diagnostic measurement of data collected from student interactions with complex learning environments. This work utilizes ECD to analyze a corpus of elementary student writings and drawings collected with a digital science notebook. First, a competency model representing the core concepts of each exercise, as well as the curricular unit as a whole, was constructed. Then, evidence models were created to map between student written and drawn artifacts and the shared competency model. Finally, the scores obtained using the evidence models were used to train a deep-learning based model for automated writing assessment, as well as to develop an automated drawing assessment model using topological abstraction. The findings reveal that ECD provides an expressive unified framework for multimodal assessment of science learning with accurate predictions of student learning.}, booktitle={Intelligent Tutoring Systems}, publisher={Springer International Publishing}, author={Smith, Andy and Aksit, Osman and Min, Wookhee and Wiebe, Eric and Mott, Bradford W. and Lester, James C.}, editor={Micarelli, A. and Stamper, J. and Panourgia, K.Editors}, year={2016}, pages={165–175} }
 @inproceedings{smith_aksit_min_wiebe_mott_lester_2016, title={Integrating real-time drawing and writing diagnostic models: An evidence-centered design framework for multimodal science assessment}, volume={0684}, booktitle={Intelligent tutoring systems, its 2016}, author={Smith, A. and Aksit, O. and Min, W. and Wiebe, E. and Mott, B. W. and Lester, J. C.}, year={2016}, pages={165–175} }
 @article{min_frankosky_mott_rowe_wiebe_boyer_lester_2015, title={DeepStealth: Leveraging Deep Learning Models for Stealth Assessment in Game-Based Learning Environments}, volume={9112}, ISBN={["978-3-319-19772-2"]}, ISSN={["1611-3349"]}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-84949009361&partnerID=MN8TOARS}, DOI={10.1007/978-3-319-19773-9_28}, abstractNote={A distinctive feature of intelligent game-based learning environments is their capacity for enabling stealth assessment. Stealth assessments gather information about student competencies in a manner that is invisible, and enable drawing valid inferences about student knowledge. We present a framework for stealth assessment that leverages deep learning, a family of machine learning methods that utilize deep artificial neural networks, to infer student competencies in a game-based learning environment for middle grade computational thinking, Engage. Students’ interaction data, collected during a classroom study with Engage, as well as prior knowledge scores, are utilized to train deep networks for predicting students’ post-test performance. Results indicate deep networks that are pre-trained using stacked denoising autoencoders achieve high predictive accuracy, significantly outperforming standard classification techniques such as support vector machines and naïve Bayes. The findings suggest that deep learning shows considerable promise for automatically inducing stealth assessment models for intelligent game-based learning environments.}, journal={ARTIFICIAL INTELLIGENCE IN EDUCATION, AIED 2015}, author={Min, Wookhee and Frankosky, Megan H. and Mott, Bradford W. and Rowe, Jonathan P. and Wiebe, Eric and Boyer, Kristy Elizabeth and Lester, James C.}, year={2015}, pages={277–286} }
 @inproceedings{buffum_frankosky_boyer_wiebe_mott_lester_2015, title={Leveraging collaboration to improve gender equity in a game-based learning environment for middle school computer science}, DOI={10.1109/respect.2015.7296496}, abstractNote={Game-based learning environments can deliver robust learning gains and also have a unique capacity to engage students. Yet they can unintentionally disadvantage students with less prior gaming experience. This is especially concerning in computer science education, as certain underrepresented groups (such as female students) may on average have less prior experience with games. This paper presents evidence that a collaborative gameplay approach can successfully address this problem at the middle school level. In an iterative, designed-based research study, we first used an experimental pilot study to investigate the nature of collaboration in the Engage game-based learning environment, and then deployed Engage in a full classroom study to measure its effectiveness at serving all students. In earlier phases of the intervention, male students outpaced their female peers in learning gains. However, female students caught up during a multi-week classroom implementation. These findings provide evidence that a collaborative gameplay approach may, over time, compensate for gender differences in experience and lead to equitable learning experiences within game-based learning environments for computer science education.}, booktitle={2015 Research in Equity and Sustained Participation in Engineering, Computing, and Technology (RESPECT)}, author={Buffum, P. S. and Frankosky, M. and Boyer, K. E. and Wiebe, Eric and Mott, B. and Lester, J.}, year={2015} }
 @article{buffum_boyer_wiebe_mott_lester_2015, title={Mind the Gap: Improving Gender Equity in Game-Based Learning Environments with Learning Companions}, volume={9112}, ISBN={["978-3-319-19772-2"]}, ISSN={["1611-3349"]}, DOI={10.1007/978-3-319-19773-9_7}, abstractNote={Game-based learning environments hold great promise for engaging learners. Yet game mechanics can initially pose barriers for students with less prior gaming experience. This paper examines game-based learning for a population of middle school learners in the US, where female students tend to have less gaming experience than male students. In a pilot study with an early version of Engage, a game-based learning environment for middle school computer science education, female students reported higher initial frustration. To address this critical issue, we developed a prototype learning companion designed specifically to reduce frustration through the telling of autobiographical stories. In a pilot study of two 7th grade classrooms, female students responded especially positively to the learning companion, eliminating the gender gap in reported frustration. The results suggest that introducing learning companions can directly contribute to making the benefits of game-based learning equitable for all learners.}, journal={ARTIFICIAL INTELLIGENCE IN EDUCATION, AIED 2015}, author={Buffum, Philip Sheridan and Boyer, Kristy Elizabeth and Wiebe, Eric N. and Mott, Bradford W. and Lester, James C.}, year={2015}, pages={64–73} }
 @article{leeman-munk_smith_mott_wiebe_lester_2015, title={Two Modes Are Better Than One: A Multimodal Assessment Framework Integrating Student Writing and Drawing}, volume={9112}, ISBN={["978-3-319-19772-2"]}, ISSN={["1611-3349"]}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-84948972893&partnerID=MN8TOARS}, DOI={10.1007/978-3-319-19773-9_21}, abstractNote={We are beginning to see the emergence of advanced automated assessment techniques that evaluate expressive student artifacts such as free-form written responses and sketches. These approaches have largely operated individually, each considering only a single mode. We hypothesize that there are synergies to be leveraged in multimodal assessments that can integrate multiple modalities of student responses to create a more complete and accurate picture of a student’s knowledge. In this paper, we introduce a novel multimodal assessment framework that integrates two techniques for automatically analyzing student artifacts: a deep learning-based model for assessing student writing, and a topology-based model for assessing student drawing. An evaluation of the framework with elementary students’ writing and drawing assessments demonstrate that 1) each of the framework’s two modalities provides an independent and complementary measure of student science learning, and 2) together, the multimodal framework significantly outperforms either uni-modal approach individually, demonstrating the potential synergistic benefits of multimodal assessment.}, journal={ARTIFICIAL INTELLIGENCE IN EDUCATION, AIED 2015}, author={Leeman-Munk, Samuel and Smith, Andy and Mott, Bradford and Wiebe, Eric and Lester, James}, year={2015}, pages={205–215} }
 @article{lee_rowe_mott_lester_2014, title={A Supervised Learning Framework for Modeling Director Agent Strategies in Educational Interactive Narrative}, volume={6}, ISSN={["1943-0698"]}, DOI={10.1109/tciaig.2013.2292010}, abstractNote={Computational models of interactive narrative offer significant potential for creating educational game experiences that are procedurally tailored to individual players and support learning. A key challenge posed by interactive narrative is devising effective director agent models that dynamically sequence story events according to players' actions and needs. In this paper, we describe a supervised machine-learning framework to model director agent strategies in an educational interactive narrative Crystal Island. Findings from two studies with human participants are reported. The first study utilized a Wizard-of-Oz paradigm where human “wizards” directed participants through Crystal Island's mystery storyline by dynamically controlling narrative events in the game environment. Interaction logs yielded training data for machine learning the conditional probabilities of a dynamic Bayesian network (DBN) model of the human wizards' directorial actions. Results indicate that the DBN model achieved significantly higher precision and recall than naive Bayes and bigram model techniques. In the second study, the DBN director agent model was incorporated into the runtime version of Crystal Island, and its impact on students' narrative-centered learning experiences was investigated. Results indicate that machine-learning director agent strategies from human demonstrations yield models that positively shape players' narrative-centered learning and problem-solving experiences.}, number={2}, journal={IEEE TRANSACTIONS ON COMPUTATIONAL INTELLIGENCE AND AI IN GAMES}, author={Lee, Seung Y. and Rowe, Jonathan P. and Mott, Bradford W. and Lester, James C.}, year={2014}, month={Jun}, pages={203–215} }
 @article{lester_spires_nietfeld_minogue_mott_lobene_2014, title={Designing game-based learning environments for elementary science education: A narrative-centered learning perspective}, volume={264}, ISSN={["1872-6291"]}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-84894486887&partnerID=MN8TOARS}, DOI={10.1016/j.ins.2013.09.005}, abstractNote={Game-based learning environments hold significant promise for STEM education, yet they are enormously complex. Crystal Island: Uncharted Discovery, is a game-based learning environment designed for upper elementary science education that has been under development in our laboratory for the past four years. This article discusses curricular and narrative interaction design requirements, presents the design of the Crystal Island learning environment, and describes its evolution through a series of pilots and field tests. Additionally, a classroom integration study was conducted to initiate a shift towards ecological validity. Results indicated that Crystal Island produced significant learning gains on both science content and problem-solving measures. Importantly, gains were consistent for gender across studies. This finding is key in light of past studies that revealed disproportionate participation by boys within game-based learning environments.}, journal={INFORMATION SCIENCES}, author={Lester, James C. and Spires, Hiller A. and Nietfeld, John L. and Minogue, James and Mott, Bradford W. and Lobene, Eleni V.}, year={2014}, month={Apr}, pages={4–18} }
 @inproceedings{baikadi_rowe_mott_lester_2014, title={Generalizability of goal recognition models in narrative-centered learning environments}, volume={8538}, DOI={10.1007/978-3-319-08786-3_24}, abstractNote={Recent years have seen growing interest in automated goal recognition. In user-adaptive systems, goal recognition is the problem of recognizing a user’s goals by observing the actions the user performs. Models of goal recognition can support student learning in intelligent tutoring systems, enhance communication efficiency in dialogue systems, or dynamically adapt software to users’ interests. In this paper, we describe an approach to goal recognition that leverages Markov Logic Networks (MLNs)—a machine learning framework that combines probabilistic inference with first-order logical reasoning—to encode relations between problem-solving goals and discovery events, domain-specific representations of user progress in narrative-centered learning environments. We investigate the impact of discovery event representations on goal recognition accuracy and efficiency. We also investigate the generalizability of discovery event-based goal recognition models across two corpora from students interacting with two distinct narrative-centered learning environments. Empirical results indicate that discovery event-based models outperform previous state-of-the-art approaches on both corpora.}, booktitle={User modeling, adaptation, and personalization, umap 2014}, author={Baikadi, A. and Rowe, J. and Mott, B. and Lester, J.}, year={2014}, pages={278–289} }
 @book{min_mott_rowe_lester_2014, title={Leveraging semi-supervised learning to predict student problem-solving performance in narrative-centered learning environments}, volume={8474 LNCS}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-84958543350&partnerID=MN8TOARS}, DOI={10.1007/978-3-319-07221-0_99}, abstractNote={This paper presents a semi-supervised machine-learning approach to predicting whether students will be successful in solving problem-solving tasks within narrative-centered learning environments. Results suggest the approach often outperforms standard supervised learning methods.}, journal={Lecture Notes in Computer Science (including subseries Lecture Notes in Artificial Intelligence and Lecture Notes in Bioinformatics)}, author={Min, Wookhee and Mott, B.W. and Rowe, J.P. and Lester, J.C.}, year={2014}, pages={664–665} }
 @inproceedings{rowe_lobene_mott_lester_2014, title={Serious games go informal: a museum-centric perspective on intelligent game-based learning}, volume={8474}, DOI={10.1007/978-3-319-07221-0_51}, abstractNote={Intelligent game-based learning environments show considerable promise for creating effective and engaging learning experiences that are tailored to individuals. To date, much of the research on intelligent game-based learning environments has focused on formal education settings and training. However, intelligent game-based learning environments also offer significant potential for informal education settings, such as museums and science centers. In this paper, we describe Future Worlds, a prototype game-based learning environment for collaborative explorations of sustainability in science museums. We report findings from a study investigating the influence of individual differences on learning and engagement in Future Worlds. Results indicate that learners showed significant gains in sustainability knowledge as well as high levels of engagement. Boys were observed to actively engage with Future Worlds for significantly longer than girls, and young children engaged with the exhibit longer than older children. These findings support the promise of intelligent game-based learning environments that dynamically recognize and adapt to learners’ individual differences during museum learning.}, booktitle={Intelligent tutoring systems, its 2014}, author={Rowe, J. P. and Lobene, E. V. and Mott, B. W. and Lester, J. C.}, year={2014}, pages={410–415} }
 @article{mcquiggan_mott_lester_2008, title={Modeling self-efficacy in intelligent tutoring systems: An inductive approach}, volume={18}, ISSN={["1573-1391"]}, DOI={10.1007/s11257-007-9040-y}, number={1-2}, journal={USER MODELING AND USER-ADAPTED INTERACTION}, author={McQuiggan, Scott W. and Mott, Bradford W. and Lester, James C.}, year={2008}, month={Feb}, pages={81–123} }
 @article{mott_lester_2006, title={Narrative-centered tutorial planning for inquiry-based learning environments}, DOI={10.1007/11774303_67}, abstractNote={Recent years have seen growing interest in narrative-centered learning environments. Leveraging the inherent structure of narrative, narrative-centered learning environments offer significant potential for inquiry-based learning in which students actively participate in engaging story-based problem-solving. A key challenge posed by narrative-centered learning is orchestrating all of the events in the unfolding story to motivate students and promote effective learning. In this paper we present a narrative-centered tutorial planning architecture that integrates narrative planning and pedagogical control. The architecture continually constructs and updates narrative plans to support the hypothesis-generation-testing cycles that form the basis for inquiry-based learning. It is being used to implement a prototype narrative-centered inquiry-based learning environment for the domain of microbiology. The planner dynamically balances narrative and pedagogical goals while at the same time satisfying the real-time constraints of highly interactive learning environments.}, number={4053}, journal={Lecture Notes in Computer Science}, author={Mott, B. W. and Lester, J. C.}, year={2006}, pages={675–684} }
 @inproceedings{ocumpaugh_andres_baker_defalco_paquette_rowe_mott_lester_georgoulas_brawner_et al., title={Affect dynamics in military trainees using vMedic: From engaged concentration to boredom to confusion}, volume={10331}, booktitle={Artificial intelligence in education, aied 2017}, author={Ocumpaugh, J. and Andres, J. M. and Baker, R. and DeFalco, J. and Paquette, L. and Rowe, J. and Mott, B. and Lester, J. and Georgoulas, V. and Brawner, K. and et al.}, pages={238–249} }
 @inproceedings{smith_min_mott_lester, title={Diagrammatic student models: Modeling student drawing performance with deep learning}, volume={9146}, booktitle={User modeling, adaptation and personalization}, author={Smith, A. and Min, W. and Mott, B. W. and Lester, J. C.}, pages={216–227} }
 @inproceedings{grafsgaard_lee_mott_boyer_lester, title={Modeling self-efficacy across age groups with automatically tracked facial expression}, volume={9112}, booktitle={Artificial intelligence in education, aied 2015}, author={Grafsgaard, J. F. and Lee, S. Y. and Mott, B. W. and Boyer, K. E. and Lester, J. C.}, pages={582–585} }