@inproceedings{skripchuk_bennett_zhang_li_price_2023, title={Analysis of Novices' Web-Based Help-Seeking Behavior While Programming}, url={https://doi.org/10.1145/3545945.3569852}, DOI={10.1145/3545945.3569852}, abstractNote={Web-based help-seeking -- finding and utilizing websites to solve a problem -- is a critical skill during programming in both professional and academic settings. However, little work has explored how students, especially novices, engage in web-based help-seeking during programming, or what strategies they use and barriers they face. This study begins to investigate these questions through analysis of students' web-search behaviors during programming. We collected think-aloud, screen recording, and log data as students completed a challenging programming task. Students were encouraged to use the web for help when needed, as if in an internship. We then qualitatively analyzed the data to address three research questions: 1) What events motivate students to use web search? 2) What strategies do students employ to search for, select, and learn from web pages? 3) What barriers do students face in web search, and when do they arise? Our results suggest that that novices use a variety of web-search strategies -- some quite unexpected -- with varying degrees of success, suggesting that web search can be a challenging skill for novice programmers. We discuss how these results inform future research and pedagogy focused on how to support students in effective web search.}, author={Skripchuk, James and Bennett, Neil and Zhang, Jeffrey and Li, Eric and Price, Thomas}, year={2023}, month={Mar} }
 @inproceedings{harred_barnes_fisk_akram_price_yoder_2023, title={Do Intentions to Persist Predict Short-Term Computing Course Enrollments}, url={https://doi.org/10.1145/3545945.3569875}, DOI={10.1145/3545945.3569875}, abstractNote={A key goal of many computer science education efforts is to increase the number and diversity of students who persist in the field of computer science and into computing careers. Many interventions have been developed in computer science designed to increase students' persistence in computing. However, it is often difficult to measure the efficacy of such interventions, as measuring actual persistence by tracking student enrollments and career placements after an intervention is difficult and time-consuming, and sometimes even impossible. In the social sciences, attitudinal research is often used to solve this problem, as attitudes can be collected in survey form around the same time that interventions are introduced and are predictive of behavior. This can allow researchers to assess the potential efficacy of an intervention before devoting the time and energy to conduct a longitudinal analysis. In this paper, we develop and validate a scale to measure intentions to persist in computing, and demonstrate its use in predicting actual persistence as defined by enrolling in another computer science course within two semesters. We conduct two analyses to do this: First, we develop a computing persistence index and test whether our scale has high alpha reliability and whether our scale predicts actual persistence in computing using students' course enrollments. Second, we conduct analyses to reduce the number of items in the scale, to make the scale easy for others to include in their own research. This paper contributes to research on computing education by developing and validating a novel measure of intentions to persist in computing, which can be used by computer science educators to evaluate potential interventions. This paper also creates a short version of the index, to ease implementation.}, author={Harred, Rachel and Barnes, Tiffany and Fisk, Susan R. and Akram, Bita and Price, Thomas W. and Yoder, Spencer}, year={2023}, month={Mar} }
 @article{marwan_akram_barnes_price_2022, title={Adaptive Immediate Feedback for Block-Based Programming: Design and Evaluation}, volume={15}, ISSN={["1939-1382"]}, url={https://doi.org/10.1109/TLT.2022.3180984}, DOI={10.1109/TLT.2022.3180984}, abstractNote={Theories on learning show that formative feedback that is immediate, specific, corrective, and positive is essential to improve novice students’ motivation and learning. However, most prior work on programming feedback focuses on highlighting student's mistakes, or detecting failed test cases after they submit a solution. In this article, we present our adaptive immediate feedback (AIF) system, which uses a hybrid data-driven feedback generation algorithm to provide students with information on their progress, code correctness, and potential errors, as well as encouragement in the middle of programming. We also present an empirical controlled study using the AIF system across several programming tasks in a CS0 classroom. Our results show that the AIF system improved students’ performance, and the proportion of students who fully completed the programming assignments, indicating increased persistence. Our results suggest that the AIF system has potential to scalably support students by giving them real-time formative feedback and the encouragement they need to complete assignments.}, number={3}, journal={IEEE TRANSACTIONS ON LEARNING TECHNOLOGIES}, publisher={Institute of Electrical and Electronics Engineers (IEEE)}, author={Marwan, Samiha and Akram, Bita and Barnes, Tiffany and Price, Thomas W.}, year={2022}, month={Jun}, pages={406–420} }
 @article{wang_le meur_bobbadi_akram_barnes_martens_price_2022, title={Exploring Design Choices to Support Novices' Example Use During Creative Open-Ended Programming}, DOI={10.1145/3478431.3499374}, abstractNote={Open-ended programming engages students by connecting computing with their real-world experience and personal interest. However, such open-ended programming tasks can be challenging, as they require students to implement features that they may be unfamiliar with. Code examples help students to generate ideas and implement program features, but students also encounter many learning barriers when using them. We explore how to design code examples to support novices' effective example use by presenting our experience of building and deploying Example Helper, a system that supports students with a gallery of code examples during open-ended programming. We deployed Example Helper in an undergraduate CS0 classroom to investigate students' example usage experience, finding that students used different strategies to browse, understand, experiment with, and integrate code examples, and that students who make more sophisticated plans also used more examples in their projects.}, journal={PROCEEDINGS OF THE 53RD ACM TECHNICAL SYMPOSIUM ON COMPUTER SCIENCE EDUCATION (SIGCSE 2022), VOL 1}, author={Wang, Wengran and Le Meur, Audrey and Bobbadi, Mahesh and Akram, Bita and Barnes, Tiffany and Martens, Chris and Price, Thomas}, year={2022}, pages={619–625} }
 @article{skripchuk_shi_price_2022, title={Identifying Common Errors in Open-Ended Machine Learning Projects}, volume={1}, url={http://dx.doi.org/10.1145/3478431.3499397}, DOI={10.1145/3478431.3499397}, journal={PROCEEDINGS OF THE 53RD ACM TECHNICAL SYMPOSIUM ON COMPUTER SCIENCE EDUCATION (SIGCSE 2022), VOL 1}, publisher={ACM}, author={Skripchuk, James and Shi, Yang and Price, Thomas}, year={2022}, pages={216–222} }
 @article{marwan_price_2022, title={iSnap: Evolution and Evaluation of a Data-Driven Hint System for Block-based Programming}, url={https://doi.org/10.1109/TLT.2022.3223577}, DOI={10.1109/TLT.2022.3223577}, abstractNote={Novice programmers often struggle on assignments, and timely help, such as a hint on what to do next, can help students continue to progress and learn, rather than giving up. However, in large programming classrooms, it is hard for instructors to provide such real-time support for every student. Researchers have therefore put tremendous effort into developing algorithms to generate automated, data-driven hints to help students at scale. Despite this, few controlled studies have directly evaluated the impact of such hints on students' performance, and learning. It is also unclear what specific design features make hints more or less effective. In this work, we present iSnap, a block-based programming environment that provides novices with data-driven, next-step hints in real-time. This paper describes our improvements to iSnap over 4 years, including its “enhanced” next-step hints with three design features: textual explanations, self-explanation prompts and an adaptive hint display. Moreover, we conducted a controlled study in an authentic classroom setting over several weeks to evaluate the impact of iSnap's enhanced hints on students' performance and learning. We found students who received the enhanced hints perform better on in-class assignments and have higher programming efficiency in homework assignments than those who did not receive hints, but that hints did not significantly impact students' learning. We also discuss the challenges of classroom studies and the implications of enhanced hints compared to prior evaluations in laboratory settings, which is essential to validate the efficacy of next-step hints' impact in a real classroom experience.}, journal={IEEE Transactions on Learning Technologies}, author={Marwan, Samiha and Price, Thomas W.}, year={2022} }
 @inproceedings{milliken_wang_cateté_martin_gomes_dong_harred_isvik_barnes_price_et al._2021, title={PlanIT! A New Integrated Tool to Help Novices Design for Open-ended Projects}, url={http://dx.doi.org/10.1145/3408877.3432552}, DOI={10.1145/3408877.3432552}, abstractNote={Project-based learning can encourage and motivate students to learn through exploring their own interests, but introduces special challenges for novice programmers. Recent research has shown that novice students perceive themselves to be bad at programming, especially when they do not know how to start writing a program, or need to create a plan before getting started. In this paper, we present PlanIT, a guided planning tool integrated with the Snap! programming environment designed to help novices plan and program their open-ended projects. Within PlanIT, students can add a description for their project, use a to do list to help break down the steps of implementation, plan important elements of their program including actors, variables, and events, and view related example projects. We report findings from a pilot study of high school students using PlanIT, showing that students who used the tool learned to make more specific and actionable plans. Results from student interviews show they appreciate the guidance that PlanIT provides, as well as the affordances it offers to more quickly create program elements.}, booktitle={Proceedings of the 52nd ACM Technical Symposium on Computer Science Education}, publisher={ACM}, author={Milliken, Alexandra and Wang, Wengran and Cateté, Veronica and Martin, Sarah and Gomes, Neeloy and Dong, Yihuan and Harred, Rachel and Isvik, Amy and Barnes, Tiffany and Price, Thomas and et al.}, year={2021}, month={Mar}, pages={232–238} }
 @article{card_wang_martens_price_2021, title={Scaffolding Game Design: Towards Tool Support for Planning Open-Ended Projects in an Introductory Game Design Class}, ISSN={["1943-6092"]}, DOI={10.1109/VL/HCC51201.2021.9576209}, abstractNote={One approach to teaching game design to students with a wide variety of disciplinary backgrounds is through team game projects that span multiple weeks, up to an entire term. However, open-ended, creative projects introduce a gamut of challenges to novice programmers. Our goal is to assist game design students with the planning stage of their projects. This paper describes our data collection process through three course interventions and student interviews, and subsequent analysis in which we learned students had difficulty expressing their creative vision and connecting the game mechanics to the intended player experience. We present these results as a step towards the goal of scaffolding the planning process for student game projects, supporting more creative ideas, clearer communication among team members, and a stronger understanding of human-centered design in software development.}, journal={2021 IEEE SYMPOSIUM ON VISUAL LANGUAGES AND HUMAN-CENTRIC COMPUTING (VL/HCC 2021)}, author={Card, Alexander and Wang, Wengran and Martens, Chris and Price, Thomas}, year={2021} }
 @article{shi_shah_wang_marwan_penmetsa_price_2021, title={Toward Semi-Automatic Misconception Discovery Using Code Embeddings}, url={http://dx.doi.org/10.1145/3448139.3448205}, DOI={10.1145/3448139.3448205}, abstractNote={Understanding students' misconceptions is important for effective teaching and assessment. However, discovering such misconceptions manually can be time-consuming and laborious. Automated misconception discovery can address these challenges by highlighting patterns in student data, which domain experts can then inspect to identify misconceptions. In this work, we present a novel method for the semi-automated discovery of problem-specific misconceptions from students' program code in computing courses, using a state-of-the-art code classification model. We trained the model on a block-based programming dataset and used the learned embedding to cluster incorrect student submissions. We found these clusters correspond to specific misconceptions about the problem and would not have been easily discovered with existing approaches. We also discuss potential applications of our approach and how these misconceptions inform domain-specific insights into students' learning processes.}, journal={LAK21 CONFERENCE PROCEEDINGS: THE ELEVENTH INTERNATIONAL CONFERENCE ON LEARNING ANALYTICS & KNOWLEDGE}, publisher={ACM}, author={Shi, Yang and Shah, Krupal and Wang, Wengran and Marwan, Samiha and Penmetsa, Poorvaja and Price, Thomas W.}, year={2021}, pages={606–612} }
 @inproceedings{marwan_gao_fisk_price_barnes_2020, title={Adaptive Immediate Feedback Can Improve Novice Programming Engagement and Intention to Persist in Computer Science}, booktitle={Proceedings of the International Computing Education Research Conference}, author={Marwan, Samiha and Gao, Ge and Fisk, Susan and Price, Thomas W. and Barnes, Tiffany}, year={2020}, pages={1–10} }
 @inbook{price_marwan_winters_williams_2020, title={An Evaluation of Data-Driven Programming Hints in a Classroom Setting}, url={https://doi.org/10.1007/978-3-030-52240-7_45}, DOI={10.1007/978-3-030-52240-7_45}, abstractNote={Data-driven programming hints are a scalable way to support students when they are stuck by automatically offering suggestions and identifying errors. However, few classroom studies have investigated data-driven hints’ impact on students’ performance and learning. In this work, we ran a controlled experiment with 241 students in an authentic classroom setting, comparing students who learned with and without hints. We found no evidence that hints improved student performance or learning overall, and we discuss possible reasons why.}, booktitle={Lecture Notes in Computer Science}, publisher={Springer International Publishing}, author={Price, Thomas W. and Marwan, Samiha and Winters, Michael and Williams, Joseph Jay}, year={2020}, pages={246–251} }
 @inproceedings{wang_zhi_milliken_lytle_price_2020, title={Crescendo : Engaging Students to Self-Paced Programming Practices}, booktitle={Proceedings of the ACM Technical Symposium on Computer Science Education}, author={Wang, Wengran and Zhi, Rui and Milliken, Alexandra and Lytle, Nicholas and Price, Thomas W}, year={2020} }
 @inproceedings{price_williams_solyst_marwan_2020, place={Honolulu, HI, USA}, title={Engaging Students with Instructor Solutions in Online Programming Homework}, booktitle={ACM CHI Conference on Human Factors in Computing Systems}, author={Price, Thomas W. and Williams, Joseph Jay and Solyst, Jaemarie and Marwan, Samiha}, year={2020} }
 @article{price_williams_solyst_marwan_2020, title={Engaging Students with Instructor Solutions in Online Programming Homework}, DOI={10.1145/3313831.3376857}, abstractNote={Students working on programming homework do not receive the same level of support as in the classroom, relying primarily on automated feedback from test cases. One low-effort way to provide more support is by prompting students to compare their solution to an instructor's solution, but it is unclear the best way to design such prompts to support learning. We designed and deployed a randomized controlled trial during online programming homework, where we provided students with an instructor's solution, and randomized whether they were prompted to compare their solution to the instructor's, to fill in the blanks for a written explanation of the instructor's solution, to do both, or neither. Our results suggest that these prompts can effectively engage students in reflecting on instructor solutions, although the results point to design trade-offs between the amount of effort that different prompts require from students and instructors, and their relative impact on learning.}, journal={PROCEEDINGS OF THE 2020 CHI CONFERENCE ON HUMAN FACTORS IN COMPUTING SYSTEMS (CHI'20)}, author={Price, Thomas W. and Williams, Joseph Jay and Solyst, Jaemarie and Marwan, Samiha}, year={2020} }
 @inproceedings{wang_rao_zhi_marwan_gao_price_2020, title={Step Tutor: Supporting Students through Step-by-Step Example-Based Feedback}, booktitle={Proceedings of the International Conference on Innovation and Technology in Computer Science Education}, author={Wang, Wengran and Rao, Yudong and Zhi, Rui and Marwan, Samiha and Gao, Ge and Price, Thomas W.}, year={2020} }
 @inproceedings{unproductive help-seeking in programming: what it is and how to address it_2020, url={http://dx.doi.org/10.1145/3341525.3387394}, DOI={10.1145/3341525.3387394}, abstractNote={While programming, novices often lack the ability to effectively seek help, such as when to ask for a hint or feedback. Students may avoid help when they need it, or abuse help to avoid putting in effort, and both behaviors can impede learning. In this paper we present two main contributions. First, we investigated log data from students working in a programming environment that offers automated hints, and we propose a taxonomy of unproductive help-seeking behaviors in programming. Second, we used these findings to design a novel user interface for hints that subtly encourages students to seek help with the right frequency, estimated with a data-driven algorithm. We conducted a pilot study to evaluate our data-driven (DD) hint display, compared to a traditional interface, where students request hints on-demand as desired. We found students with the DD display were less than half as likely to engage in unproductive help-seeking, and we found suggestive evidence that this may improve their learning.}, booktitle={Proceedings of the 2020 ACM Conference on Innovation and Technology in Computer Science Education}, year={2020}, month={Jun} }
 @article{price_dong_zhi_paaßen_lytle_cateté_barnes_2019, title={A Comparison of the Quality of Data-Driven Programming Hint Generation Algorithms}, volume={29}, ISSN={1560-4292 1560-4306}, url={http://dx.doi.org/10.1007/s40593-019-00177-z}, DOI={10.1007/s40593-019-00177-z}, number={3}, journal={International Journal of Artificial Intelligence in Education}, publisher={Springer Science and Business Media LLC}, author={Price, Thomas W. and Dong, Yihuan and Zhi, Rui and Paaßen, Benjamin and Lytle, Nicholas and Cateté, Veronica and Barnes, Tiffany}, year={2019}, month={May}, pages={368–395} }
 @inproceedings{marwan_jay williams_price_2019, place={New York, NY, USA}, title={An Evaluation of the Impact of Automated Programming Hints on Performance and Learning}, url={https://doi.org/10.1145/3291279.3339420}, DOI={10.1145/3291279.3339420}, abstractNote={A growing body of work has explored how to automatically generate hints for novice programmers, and many programming environments now employ these hints. However, few studies have investigated the efficacy of automated programming hints for improving performance and learning, how and when novices find these hints beneficial, and the tradeoffs that exist between different types of hints. In this work, we explored the efficacy of next-step code hints with 2 complementary features: textual explanations and self-explanation prompts. We conducted two studies in which novices completed two programming tasks in a block-based programming environment with automated hints. In Study 1, 10 undergraduate students completed 2 programming tasks with a variety of hint types, and we interviewed them to understand their perceptions of the affordances of each hint type. For Study 2, we recruited a convenience sample of participants without programming experience from Amazon Mechanical Turk. We conducted a randomized experiment comparing the effects of hints' types on learners' performance and performance on a subsequent task without hints. We found that code hints with textual explanations significantly improved immediate programming performance. However, these hints only improved performance in a subsequent post-test task with similar objectives, when they were combined with self-explanation prompts. These results provide design insights into how automatically generated code hints can be improved with textual explanations and prompts to self-explain, and provide evidence about when and how these hints can improve programming performance and learning.}, booktitle={Proceedings of the 2019 ACM Conference on International Computing Education Research}, publisher={Association for Computing Machinery}, author={Marwan, Samiha and Jay Williams, Joseph and Price, Thomas}, year={2019}, pages={61–70} }
 @inproceedings{dong_marwan_catete_price_barnes_2019, place={New York, NY, USA}, title={Defining Tinkering Behavior in Open-ended Block-based Programming Assignments}, ISBN={9781450358903}, url={http://dx.doi.org/10.1145/3287324.3287437}, DOI={10.1145/3287324.3287437}, abstractNote={Tinkering has been shown to have a positive influence on students in open-ended making activities. Open-ended programming assignments in block-based programming resemble making activities in that both of them encourage students to tinker with tools to create their own solutions to achieve a goal. However, previous studies of tinkering in programming discussed tinkering as a broad, ambiguous term, and investigated only self-reported data. To our knowledge, no research has studied student tinkering behaviors while solving problems in block-based programming environments. In this position paper, we propose a definition for tinkering in block-based programming environments as a kind of behavior that students exhibit when testing, exploring, and struggling during problem-solving. We introduce three general categories of tinkering behaviors (test-based, prototype-based, and construction-based tinkering) derived from student data, and use case studies to demonstrate how students exhibited these behaviors in problem-solving. We created the definitions using a mixed-methods research design combining a literature review with data-driven insights from submissions of two open-ended programming assignments in iSnap, a block-based programming environment. We discuss the implication of each type of tinkering behavior for learning. Our study and results are the first in this domain to define tinkering based on student behaviors in a block-based programming environment.}, booktitle={Proceedings of the 50th ACM Technical Symposium on Computer Science Education  - SIGCSE '19}, publisher={ACM Press}, author={Dong, Yihuan and Marwan, Samiha and Catete, Veronica and Price, Thomas and Barnes, Tiffany}, year={2019}, pages={1204–1210} }
 @inproceedings{zhi_chi_barnes_price_2019, place={New York, NY, USA}, title={Evaluating the Effectiveness of Parsons Problems for Block-based Programming}, ISBN={9781450361859}, url={http://dx.doi.org/10.1145/3291279.3339419}, DOI={10.1145/3291279.3339419}, abstractNote={Parsons problems are program puzzles, where students piece together code fragments to construct a program. Similar to block-based programming environments, Parsons problems eliminate the need to learn syntax. Parsons problems have been shown to improve learning efficiency when compared to writing code or fixing incorrect code in lab studies, or as part of a larger curriculum. In this study, we directly compared Parsons problems with block-based programming assignments in classroom settings. We hypothesized that Parsons problems would improve students' programming efficiency on the lab assignments where they were used, without impacting performance on the subsequent, related homework or the later programming project. Our results confirmed our hypothesis, showing that on average Parsons problems took students about half as much time to complete compared to equivalent programming problems. At the same time, we found no evidence to suggest that students performed worse on subsequent assignments, as measured by performance and time on task. The results indicate that the effectiveness of Parsons problems is not simply based on helping students avoid syntax errors. We believe this is because Parsons problems dramatically reduce the programming solution space, letting students focus on solving the problem rather than having to solve the combined problem of devising a solution, searching for needed components, and composing them together.}, booktitle={Proceedings of the 2019 ACM Conference on International Computing Education Research  - ICER '19}, publisher={ACM Press}, author={Zhi, Rui and Chi, Min and Barnes, Tiffany and Price, Thomas W.}, year={2019}, pages={51–59} }
 @inproceedings{zhi_price_marwan_milliken_barnes_chi_2019, place={New York, NY, USA}, title={Exploring the Impact of Worked Examples in a Novice Programming Environment}, ISBN={9781450358903}, url={http://dx.doi.org/10.1145/3287324.3287385}, DOI={10.1145/3287324.3287385}, abstractNote={Research in a variety of domains has shown that viewing worked examples (WEs) can be a more efficient way to learn than solving equivalent problems. We designed a Peer Code Helper system to display WEs, along with scaffolded self-explanation prompts, in a block-based, novice programming environment called \snap. We evaluated our system during a high school summer camp with 22 students. Participants completed three programming problems with access to WEs on either the first or second problem. We found that WEs did not significantly impact students' learning, but may have impacted students' intrinsic cognitive load, suggesting that our WEs with scaffolded prompts may be an inherently different learning task. Our results show that WEs saved students time on initial tasks compared to writing code, but some of the time saved was lost in subsequent programming tasks. Overall, students with WEs completed more tasks within a fixed time period, but not significantly more. WEs may improve students' learning efficiency when programming, but these effects are nuanced and merit further study.}, booktitle={Proceedings of the 50th ACM Technical Symposium on Computer Science Education  - SIGCSE '19}, publisher={ACM Press}, author={Zhi, Rui and Price, Thomas W. and Marwan, Samiha and Milliken, Alexandra and Barnes, Tiffany and Chi, Min}, year={2019}, pages={98–104} }
 @inproceedings{marwan_lytle_williams_price_2019, place={New York, NY, USA}, title={The Impact of Adding Textual Explanations to Next-Step Hints in a Novice Programming Environment}, url={https://doi.org/10.1145/3304221.3319759}, DOI={10.1145/3304221.3319759}, abstractNote={Automated hints, a powerful feature of many programming environments, have been shown to improve students' performance and learning. New methods for generating these hints use historical data, allowing them to scale easily to new classrooms and contexts. These scalable methods often generate next-step, code hints that suggest a single edit for the student to make to their code. However, while these code hints tell the student what to do, they do not explain why, which can make these hints hard to interpret and decrease students' trust in their helpfulness. In this work, we augmented code hints by adding adaptive, textual explanations in a block-based, novice programming environment. We evaluated their impact in two controlled studies with novice learners to investigate how our results generalize to different populations. We measured the impact of textual explanations on novices' programming performance. We also used quantitative analysis of log data, self-explanation prompts, and frequent feedback surveys to evaluate novices' understanding and perception of the hints throughout the learning process. Our results showed that novices perceived hints with explanations as significantly more relevant and interpretable than those without explanations, and were also better able to connect these hints to their code and the assignment. However, we found little difference in novices' performance. Our results suggest that explanations have the potential to make code hints more useful, but it is unclear whether this translates into better overall performance and learning.}, booktitle={Proceedings of the 2019 ACM Conference on Innovation and Technology in Computer Science Education}, publisher={Association for Computing Machinery}, author={Marwan, Samiha and Lytle, Nicholas and Williams, Joseph Jay and Price, Thomas}, year={2019}, pages={520–526} }
 @inproceedings{zhi_marwan_dong_lytle_price_barnes_2019, title={Toward Data-Driven Example Feedback for Novice Programming}, booktitle={Proceedings of the International Conference on Educational Data Mining}, author={Zhi, Rui and Marwan, Samiha and Dong, Yihuan and Lytle, Nicholas and Price, Thomas W and Barnes, Tiffany}, year={2019}, pages={218–227} }
 @inproceedings{zhi_lytle_price_2018, place={New York, NY, USA}, title={Exploring Instructional Support Design in an Educational Game for K-12 Computing Education}, url={https://doi.org/10.1145/3159450.3159519}, DOI={10.1145/3159450.3159519}, abstractNote={Instructional supports (Supports) help students learn more effectively in intelligent tutoring systems and gamified educational environments. However, the implementation and success of Supports vary by environment. We explored Support design in an educational programming game, BOTS, implementing three different strategies: instructional text (Text), worked examples (Examples) and buggy code (Bugs). These strategies are adapted from promising Supports in other domains and motivated by established educational theory. We evaluated our Supports through a pilot study with middle school students. Our results suggest Bugs may be a promising strategy, as demonstrated by the lower completion time and solution code length in assessment puzzles. We end reflecting on our design decisions providing recommendations for future iterations. Our motivations, design process, and study's results provide insight into the design of Supports for programming games.}, booktitle={Proceedings of the 49th ACM Technical Symposium on Computer Science Education}, publisher={Association for Computing Machinery}, author={Zhi, Rui and Lytle, Nicholas and Price, Thomas W.}, year={2018}, pages={747–752} }
 @article{paassen_hammer_price_barnes_gross_pinkwart_2018, title={The Continuous Hint Factory - Providing Hints in Vast and Sparsely Populated Edit Distance Spaces}, volume={10}, DOI={https://doi.org/10.5281/zenodo.3554697}, abstractNote={Intelligent tutoring systems can support students in solving multi-step tasks by providing hints regarding what to do next. However, engineering such next-step hints manually or via an expert model becomes infeasible if the space of possible states is too large. Therefore, several approaches have emerged to infer next-step hints automatically, relying on past students' data. In particular, the Hint Factory (Barnes & Stamper, 2008) recommends edits that are most likely to guide students from their current state towards a correct solution, based on what successful students in the past have done in the same situation. Still, the Hint Factory relies on student data being available for any state a student might visit while solving the task, which is not the case for some learning tasks, such as open-ended programming tasks.
In this contribution we provide a mathematical framework for edit-based hint policies and, based on this theory, propose a novel hint policy to provide edit hints in vast and sparsely populated state spaces. In particular, we extend the Hint Factory by considering data of past students in all states which are similar to the student's current state and creating hints approximating the weighted average of all these reference states. Because the space of possible weighted averages is continuous, we call this approach the Continuous Hint Factory.
In our experimental evaluation, we demonstrate that the Continuous Hint Factory can predict more accurately what capable students would do compared to existing prediction schemes on two learning tasks, especially in an open-ended programming task, and that the Continuous Hint Factory is comparable to existing hint policies at reproducing tutor hints on a simple UML diagram task.}, number={1}, journal={Journal of Educational Data Mining}, author={Paassen, B. and Hammer, B. and Price, T.W. and Barnes, T. and Gross, S. and Pinkwart, N.}, year={2018}, month={Jun}, pages={1–35} }
 @inbook{price_zhi_dong_lytle_barnes_2018, title={The Impact of Data Quantity and Source on the Quality of Data-Driven Hints for Programming}, ISBN={9783319938424 9783319938431}, ISSN={0302-9743 1611-3349}, url={http://dx.doi.org/10.1007/978-3-319-93843-1_35}, DOI={10.1007/978-3-319-93843-1_35}, abstractNote={In the domain of programming, intelligent tutoring systems increasingly employ data-driven methods to automate hint generation. Evaluations of these systems have largely focused on whether they can reliably provide hints for most students, and how much data is needed to do so, rather than how useful the resulting hints are to students. We present a method for evaluating the quality of data-driven hints and how their quality is impacted by the data used to generate them. Using two datasets, we investigate how the quantity of data and the source of data (whether it comes from students or experts) impact one hint generation algorithm. We find that with student training data, hint quality stops improving after 15–20 training solutions and can decrease with additional data. We also find that student data outperforms a single expert solution but that a comprehensive set of expert solutions generally performs best.}, booktitle={Lecture Notes in Computer Science}, publisher={Springer International Publishing}, author={Price, Thomas W. and Zhi, Rui and Dong, Yihuan and Lytle, Nicholas and Barnes, Tiffany}, year={2018}, pages={476–490} }
 @article{price_2018, place={New York, NY, USA}, title={iSnap: Automatic Hints and Feedback for Block-based Programming}, url={https://doi.org/10.1145/3159450.3162202}, DOI={10.1145/3159450.3162202}, abstractNote={iSnap is a block-based programming environment that supports struggling students with on-demand hints and error-checking feedback. iSnap is an extension of Snap!, a creative and novice-friendly programming environment, used in the Beauty and Joy of Computing (BJC) AP CS Principles curriculum. iSnap is designed to support the open-ended, exploratory programming problems of BJC, while adapting to many possible student solutions. When students ask iSnap for help, it highlights possible errors in their code and suggests next steps they can make. Hints are presented visually, right alongside students/ code, making them easy to interpret and implement. iSnap/s hints are generated automatically from student data, so no teacher input is required to create them, making iSnap appropriate for both new and experienced instructors. The demonstration will showcase iSnap/s hints on a variety of assignments and explain how the algorithm is working behind the scenes to generate data-driven hints. It will also include an overview of the results from two years of research with iSnap on how students seek and use programming help. A key objective of this demonstration is to solicit feedback from SIGCSE attendees on the design of iSnap as we work to make the system ready for deployment in classrooms. More information on iSnap can be found at http://go.ncsu.edu/isnap.}, journal={SIGCSE'18: PROCEEDINGS OF THE 49TH ACM TECHNICAL SYMPOSIUM ON COMPUTER SCIENCE EDUCATION}, publisher={Association for Computing Machinery}, author={Price, Thomas W.}, year={2018}, pages={1113–1113} }
 @inproceedings{price_zhi_barnes_2017, title={Evaluation of a Data-driven Feedback Algorithm for Open-ended Programming}, booktitle={Proceedings of the International Conference on Educational Data Mining}, author={Price, Thomas W. and Zhi, Rui and Barnes, Tiffany}, year={2017} }
 @article{price_liu_catete_barnes_2017, place={New York, NY, USA}, title={Factors Influencing Students' Help-Seeking Behavior while Programming with Human and Computer Tutors}, url={https://doi.org/10.1145/3105726.3106179}, DOI={10.1145/3105726.3106179}, abstractNote={When novice students encounter difficulty when learning to program, some can seek help from instructors or teaching assistants. This one-on-one tutoring is highly effective at fostering learning, but busy instructors and large class sizes can make expert help a scarce resource. Increasingly, programming environments attempt to imitate this human support by providing students with hints and feedback. In order to design effective, computer-based help, it is important to understand how and why students seek and avoid help when programming, and how this process differs when the help is provided by a human or a computer. We explore these questions through a qualitative analysis of 15 students' interviews, in which they reflect on solving two programming problems with human and computer help. We discuss implications for help design and present hypotheses on students' help-seeking behavior.}, journal={PROCEEDINGS OF THE 2017 ACM CONFERENCE ON INTERNATIONAL COMPUTING EDUCATION RESEARCH (ICER 17)}, publisher={Association for Computing Machinery}, author={Price, Thomas W. and Liu, Zhongxiu and Catete, Veronica and Barnes, Tiffany}, year={2017}, pages={127–135} }
 @article{price_zhi_barnes_2017, title={Hint Generation Under Uncertainty: The Effect of Hint Quality on Help-Seeking Behavior}, volume={10331}, ISBN={["978-3-319-61424-3"]}, ISSN={["1611-3349"]}, url={http://dx.doi.org/10.1007/978-3-319-61425-0_26}, DOI={10.1007/978-3-319-61425-0_26}, abstractNote={Much research in Intelligent Tutoring Systems has explored how to provide on-demand hints, how they should be used, and what effect they have on student learning and performance. Most of this work relies on hints created by experts and assumes that all help provided by the tutor is correct and of high quality. However, hints may not all be of equal value, especially in open-ended problem solving domains, where context is important. This work argues that hint quality, especially when using data-driven hint generation techniques, is inherently uncertain. We investigate the impact of hint quality on students’ help-seeking behavior in an open-ended programming environment with on-demand hints. Our results suggest that the quality of the first few hints on an assignment is positively associated with future hint use on the same assignment. Initial hint quality also correlates with possible help abuse. These results have important implications for hint design and generation.}, journal={ARTIFICIAL INTELLIGENCE IN EDUCATION, AIED 2017}, author={Price, Thomas W. and Zhi, Rui and Barnes, Tiffany}, year={2017}, pages={311–322} }
 @inproceedings{price_barnes_2017, title={Position paper: Block-based programming should offer intelligent support for learners}, DOI={10.1109/blocks.2017.8120414}, abstractNote={Block-based programming environments make learning to program easier by allowing learners to focus on concepts rather than syntax. However, these environments offer little support when learners encounter difficulty with programming concepts themselves, especially in the absence of instructors. Textual programming environments increasingly use AI and data mining to provide intelligent, adaptive support for students, similar to human tutoring, which has been shown to improve performance and learning outcomes. In this position paper, we argue that block-based programming environments should also include these features. This paper gives an overview of promising research in intelligent support for programming and highlights the challenges and opportunities for applying this work to block-based programming.}, booktitle={2017 IEEE Blocks and Beyond Workshop (B&B)}, author={Price, Thomas and Barnes, T.}, year={2017}, pages={65–68} }
 @inproceedings{price_brown_piech_rivers_2017, place={New York, NY, USA}, title={Sharing and Using Programming Log Data (Abstract Only)}, url={https://doi.org/10.1145/3017680.3022366}, DOI={10.1145/3017680.3022366}, abstractNote={As more programming environments add logging features and programming data becomes more accessible, it is important to have a conversation about how we share and use this data. Uses of programming log data range from big-picture analyses to dashboards for instant teacher feedback, to intelligent, data-driven learning environments. The goal of this BOF is to talk about what data is important to collect, where it can be gathered and shared, what general data formats make sense, how to handle privacy and anonymization, and what ultimately we want to see the data used for. The BOF welcomes both producers of programming log data and current or potential consumers, interested in how it could be applied in their classrooms or research. One hopeful outcome of this BOF is a commitment to documenting and sharing existing programming data in an accessible location and format.}, booktitle={Proceedings of the 2017 ACM SIGCSE Technical Symposium on Computer Science Education}, publisher={Association for Computing Machinery}, author={Price, Thomas W. and Brown, Neil C.C. and Piech, Chris and Rivers, Kelly}, year={2017}, pages={729} }
 @inproceedings{price_dong_lipovac_2017, place={New York}, title={iSnap: Towards Intelligent Tutoring in Novice Programming Environments.}, url={http://dx.doi.org/10.1145/3017680.3017762}, DOI={10.1145/3017680.3017762}, abstractNote={Programming environments intentionally designed to support novices have become increasingly popular, and growing research supports their efficacy. While these environments offer features to engage students and reduce the burden of syntax errors, they currently offer little support to students who get stuck and need expert assistance. Intelligent Tutoring Systems (ITSs) are computer systems designed to play this role, helping and guiding students to achieve better learning outcomes. We present iSnap, an extension to the Snap programming environment which adds some key features of ITSs, including detailed logging and automatically generated hints. We share results from a pilot study of iSnap, indicating that students are generally willing to use hints and that hints can create positive outcomes. We also highlight some key challenges encountered in the pilot study and discuss their implications for future work.}, booktitle={SIGCSE '17: Proceedings of the 2017 ACM SIGCSE Technical Symposium on Computer Science Education}, publisher={Association for Computing Machinery}, author={Price, Thomas and Dong, Yihuan and Lipovac, Dragan}, year={2017}, month={Mar}, pages={483–488} }
 @inproceedings{duvall_eagle_narcisse_price_2016, place={New York, NY, USA}, title={Clashroom: A Game to Enhance the Classroom Experience (Abstract Only)}, url={https://doi.org/10.1145/2839509.2850556}, DOI={10.1145/2839509.2850556}, abstractNote={Clashroom is a web-based fantasy game that was created to enhance the motivation and learning of a traditional classroom. It was created to address several drawbacks of other educational games by being broadly applicable to any course, requiring no class time, and allowing for optional rather than mandatory game play. Each player in the game is a dragon trainer, hoping to train a champion pet dragon. To gain skills and magical items for the dragon to use, players complete educational quests set up by the teacher. Players compete in weekly tournaments, where the quest rewards are vital to success. Because the gameplay and the learning material are decoupled, the game can be used for any subject at any level. After initial testing, students report that the game is fun and that the intrinsic rewards of a fun game are motivation enough for extra learning.}, booktitle={Proceedings of the 47th ACM Technical Symposium on Computing Science Education}, publisher={Association for Computing Machinery}, author={Duvall, Shannon and Eagle, Daniel R. and Narcisse, Riese P. and Price, Thomas W.}, year={2016}, pages={692} }
 @article{price_brown_lipovac_barnes_kolling_2016, place={New York, NY, USA}, title={Evaluation of a Frame-based Programming Editor}, url={https://doi.org/10.1145/2960310.2960319}, DOI={10.1145/2960310.2960319}, abstractNote={Frame-based editing is a novel way to edit programs, which claims to combine the benefits of textual and block-based programming. It combines structured `frames' of preformatted code, designed to reduce the burden of syntax, with `slots' that allow for efficient textual entry of expressions. We present an empirical evaluation of Stride, a frame-based language used in the Greenfoot IDE. We compare two groups of middle school students who worked on a short programming activity in Greenfoot, one using the original Java editor, and one using the Stride editor. We found that the two groups reported similarly low levels of frustration and high levels of satisfaction, but students using Stride progressed through the activity more quickly and completed more objectives. The Stride group also spent significantly less time making purely syntactic edits to their code and significantly less time with non-compilable code.}, journal={PROCEEDINGS OF THE 2016 ACM CONFERENCE ON INTERNATIONAL COMPUTING EDUCATION RESEARCH (ICER'16)}, publisher={Association for Computing Machinery}, author={Price, Thomas W. and Brown, Neil C. C. and Lipovac, Dragan and Barnes, Tiffany and Kolling, Michael}, year={2016}, pages={33–42} }
 @inproceedings{price_dong_barnes_2016, title={Generating data-driven hints for open-ended programming}, booktitle={Proceedings of the 9th International Conference on Educational Data Mining, International Educational Data Mining Society}, author={Price, Thomas W and Dong, Yihuan and Barnes, Tiffany}, year={2016}, pages={191–198} }
 @inproceedings{price_cateté_albert_barnes_garcia_2016, place={New York, NY, USA}, title={Lessons Learned from "BJC" CS Principles Professional Development}, ISBN={9781450336857}, url={http://dx.doi.org/10.1145/2839509.2844625}, DOI={10.1145/2839509.2844625}, abstractNote={Computer Science Principles (CSP) will become an Advanced Placement course during the 2016-17 school year, and there is an immediate need to train new teachers to be leaders in computing classrooms. From 2012-2015, the Beauty and Joy of Computing team offered professional development (PD) to 133 teachers, resulting in 89 BJC CSP courses taught in high schools. Our data show that the PD improved teachers' confidence in our four core content categories and met its primary goal of training teachers in equitable, inquiry-based instruction. In this paper, we present the evolution of the BJC PD, its challenges and lessons that we learned while continually adapting to teachers' needs and contexts.}, booktitle={Proceedings of the 47th ACM Technical Symposium on Computing Science Education - SIGCSE '16}, publisher={ACM Press}, author={Price, Thomas W. and Cateté, Veronica and Albert, Jennifer and Barnes, Tiffany and Garcia, Daniel D.}, year={2016}, pages={467–472} }
 @article{cardona-rivera_price_winer_young_2016, title={Question Answering in the Context of Stories Generated by Computers}, volume={4}, journal={Advances in Cognitive Systems}, author={Cardona-Rivera, R. and Price, T.W. and Winer, D. and Young, R.M.}, year={2016}, month={Jun}, pages={227–245} }
 @inproceedings{price_barnes_2015, title={An Exploration of Data-Driven Hint Generation in an Open-Ended Programming Problem}, booktitle={International Workshop on Graph-Based Educational Data Mining}, author={Price, Thomas W and Barnes, Tiffany}, year={2015} }
 @inproceedings{price_lynch_barnes_chi_2015, title={An Improved Data-Driven Hint Selection Algorithm for Probability Tutors}, booktitle={The 8th International Conference on Education Data Mining}, author={Price, Thomas W and Lynch, Collin F and Barnes, Tiffany and Chi, Min}, year={2015} }
 @inproceedings{price_albert_catete_barnes_2015, title={BJC in action: Comparison of student perceptions of a computer science principles course}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-84959906858&partnerID=MN8TOARS}, DOI={10.1109/respect.2015.7296506}, abstractNote={The Beauty and Joy of Computing (BJC) is a curriculum for the new AP Computer Science Principles course. Over the past 2 years, we have collected post-course surveys from 399 students participating in the BJC course. This paper investigates how the responses of females and students from underrepresented racial minority groups (URMs) differed from those of their counterparts. We found that female students had taken fewer CS courses prior to BJC but that students from URMs had taken more prior CS courses. Both groups were nearly equally likely to recommend the course to a friend, with about 80% recommending. We found no evidence to suggest that female students showed more or less interest in specific CS topics, such as learning how computing has changed the world or making mobile apps/games. Despite having taken more CS courses prior to BJC, we found that students from URMs were overall less likely to intend to take additional CS courses. Overall, our findings are fairly consistent with the literature, and suggest that BJC makes some progress towards broadening participation in computing.}, booktitle={2015 Research in Equity and Sustained Participation in Engineering, Computing, and Technology (RESPECT)}, publisher={IEEE}, author={Price, Thomas W. and Albert, Jennifer and Catete, Veronica and Barnes, Tiffany}, year={2015}, pages={1–4} }
 @inproceedings{price_barnes_2015, place={New York, NY, USA}, title={Comparing Textual and Block Interfaces in a Novice Programming Environment}, ISBN={9781450336307}, url={http://dx.doi.org/10.1145/2787622.2787712}, DOI={10.1145/2787622.2787712}, abstractNote={Visual, block-based programming environments present an alternative way of teaching programming to novices and have proven successful in classrooms and informal learning settings. However, few studies have been able to attribute this success to specific features of the environment. In this study, we isolate the most fundamental feature of these environments, the block interface, and compare it directly to its textual counterpart. We present analysis from a study of two groups of novice programmers, one assigned to each interface, as they completed a simple programming activity. We found that while the interface did not seem to affect users' attitudes or perceived difficulty, students using the block interface spent less time off task and completed more of the activity's goals in less time.}, booktitle={Proceedings of the eleventh annual International Conference on International Computing Education Research - ICER '15}, publisher={ACM Press}, author={Price, Thomas W. and Barnes, Tiffany}, year={2015}, pages={91–99} }
 @article{price_barnes_2015, title={Creating Data-Driven Feedback for Novices in Goal-Driven Programming Projects}, volume={9112}, ISBN={["978-3-319-19772-2"]}, ISSN={["1611-3349"]}, DOI={10.1007/978-3-319-19773-9_132}, abstractNote={Programming environments that afford the creation of media-rich, goal-driven projects, such as games, stories and simulations, are effective at engaging novice users. However, the open-ended nature of these projects makes it difficult to generate ITS-style guidance for students in need of help. In domains where students produce similar, overlapping solutions, data-driven techniques can leverage the work of previous students to provide feedback. However, our data suggest that solutions to these projects have insufficient overlap to apply current data-driven methods. We propose a novel subtree-based state matching technique that will find partially overlapping solutions to generate feedback across diverse student programs. We will build a system to generate this feedback, test the technique on historical data, and evaluate the generated feedback in a study of goal-driven programming projects. If successful, this approach will provide insight into how to leverage structural similarities across complex, creative problem solutions to provide data-driven feedback for intelligent tutoring.}, journal={ARTIFICIAL INTELLIGENCE IN EDUCATION, AIED 2015}, author={Price, Thomas W. and Barnes, Tiffany}, year={2015}, pages={856–859} }
 @inproceedings{price_barnes_2015, title={Creating data-driven feedback for novices in goal-driven programming projects}, booktitle={International Conference on Artificial Intelligence in Education}, author={Price, Thomas W and Barnes, Tiffany}, year={2015}, pages={856–859} }
 @inproceedings{price_2015, place={New York, NY, USA}, title={Integrating Intelligent Feedback into Block Programming Environments}, url={https://doi.org/10.1145/2787622.2787748}, DOI={10.1145/2787622.2787748}, abstractNote={Block Programming Environments (BPEs) are becoming popular tools for introducing novices to programming, due in part to their connection with students' interests in games, apps and stories. This has led to increasing use of BPEs outside of classroom settings, where knowledgeable instructors are not always available. Intelligent Tutoring Systems (ITSs) can keep students on track in the absence of instructors by providing hints and warnings to students in need of help. Further, data-driven techniques can generate this feedback automatically from previous students' attempts at a problem. This research focuses on the integration of this data-driven, ITS-style feedback into a modern BPE and the evaluation of its impact.}, booktitle={Proceedings of the Eleventh Annual International Conference on International Computing Education Research}, publisher={Association for Computing Machinery}, author={Price, Thomas W.}, year={2015}, pages={275–276} }
 @inproceedings{zhou_price_lynch_barnes_chi_2015, title={The Impact of Granularity on Worked Examples and Problem Solving}, booktitle={Annual Meeting of the Cognitive Science Society (CogSci)}, author={Zhou, Guojing and Price, Thomas W and Lynch, Collin and Barnes, Tiffany and Chi, Min}, year={2015} }
 @inproceedings{lynch_price_chi_barnes_2015, title={Using the Hint Factory to Compare Model-based Tutoring Systems}, booktitle={International Workshop on Graph-Based Educational Data Mining}, author={Lynch, Collin and Price, Thomas W and Chi, Min and Barnes, Tiffany}, year={2015} }
 @inproceedings{price_young_2014, title={Towards an Extended Declarative Representation for Camera Planning}, booktitle={Workshops at the Twenty-Eighth AAAI Conference on Artificial Intelligence}, author={Price, Thomas William and Young, R Michael}, year={2014} }
 @article{zhou_lynch_price_barnes_chi, title={The Impact of Granularity on the Effectiveness of Students’ Pedagogical Decision}, author={Zhou, Guojing and Lynch, Collin F and Price, Thomas W and Barnes, Tiffany and Chi, Min} }