@inproceedings{king_2021, place={New York, NY, USA}, title={Combining Theory and Practice in Data Structures & Algorithms Course Projects: An Experience Report}, url={https://doi.org/10.1145/3408877.3432476}, DOI={10.1145/3408877.3432476}, abstractNote={CS2 course projects can often be too prescriptive by telling students which algorithms or data structures are necessary to efficiently solve a given problem. Students may not fully understand why these algorithms or data structures were chosen, and they may not have an opportunity to empirically observe the impact of such design decisions. In 2019, we redesigned our CS2 course projects at North Carolina State University to help demonstrate the importance of critical-thinking and analysis activities when developing software. The objective of the redesigned course project is to connect computer science theory with software development practice by incorporating algorithm design and analysis, data structure selection, and experimental analysis as part of the software development lifecycle. For the project, students first create a design proposal that incorporates algorithm design and analysis, data structure selection, and other software design tasks. Next, students implement and test their software, while teaching staff test cases impose grade penalties for incorrect and/or inefficient implementations. Finally, students perform experimental analysis to empirically observe performance differences when using different data structures. Between January 2019 and May 2020, we collected end-of-course survey responses from 202 out of 536 students (37.7% response rate). Overall, 90.6% of respondents indicated that the project helped with understanding the importance of analysis and design when developing software. In this paper, we discuss our project activities in detail, along with lessons learned and suggestions for adopting similar projects in other CS2 courses.}, booktitle={Proceedings of the 52nd ACM Technical Symposium on Computer Science Education}, publisher={Association for Computing Machinery}, author={King, Jason}, year={2021}, pages={959–965} }
 @article{heckman_schmidt_king_2020, title={Integrating Testing Throughout the CS Curriculum}, ISSN={["2159-4848"]}, DOI={10.1109/ICSTW50294.2020.00079}, abstractNote={Software testing is a critical component of any software development lifecycle, but becoming an experienced software tester requires understanding many strategies for writing high-quality test cases and a significant amount of practice. Situated learning theory suggests that students should be exposed to things they would see in a professional workplace. In terms of software testing, students should be exposed to real-world software testing practices in a variety of contexts, from the simplest of programs to the very complex. The goal of this paper is to share our experience integrating software testing into our undergraduate curriculum at North Carolina State University. In this paper, we discuss how software testing is taught in our CS1 – Introductory Programming, CS2 – Software Development Fundamentals, and several other courses beyond CS2. Over the past 10 years of teaching software testing in introductory programming courses, we discuss lessons learned and highlight open concerns for future research.}, journal={2020 IEEE 13TH INTERNATIONAL CONFERENCE ON SOFTWARE TESTING, VERIFICATION AND VALIDATION WORKSHOPS (ICSTW)}, author={Heckman, Sarah and Schmidt, Jessica Young and King, Jason}, year={2020}, pages={441–444} }
 @article{heckman_king_2018, place={New York, NY, USA}, title={Developing Software Engineering Skills using Real Tools for Automated Grading}, DOI={10.1145/3159450.3159595}, abstractNote={Situated learning theory supports engaging students with materials and resources that reflect professional standards and best practices. Starting with our introductory courses, we incorporate situated learning to support student engagement in software engineering practices and processes through the use of industrial strength open-source tools in several classes throughout the undergraduate computer science curriculum at NC State University. Additionally, these tools support several logistical and educational needs in computer science classrooms, including assignment submission systems and automated grading. In this tools paper, we present our Canary Framework for supporting software engineering practices through the use of Eclipse for development; GitHub for submission and collaboration; and Jenkins for continuous integration and automated grading. These tools are used in five of ten core courses by more than 3000 students over ten semesters. While the use of these tools in education is not unique, we want to share our model of using professional tools in a classroom setting and our experiences on how this framework can support multiple courses throughout the curriculum and at scale.}, journal={SIGCSE'18: PROCEEDINGS OF THE 49TH ACM TECHNICAL SYMPOSIUM ON COMPUTER SCIENCE EDUCATION}, publisher={ACM}, author={Heckman, Sarah and King, Jason}, year={2018}, pages={794–799} }
 @inproceedings{carver_burcham_kocak_bener_felderer_gander_king_markkula_oivo_sauerwein_et al._2016, place={New York, NY, USA}, title={Establishing a Baseline for Measuring Advancement in the Science of Security: An Analysis of the 2015 IEEE Security & Privacy Proceedings}, url={http://doi.acm.org/10.1145/2898375.2898380}, DOI={10.1145/2898375.2898380}, abstractNote={To help establish a more scientific basis for security science, which will enable the development of fundamental theories and move the field from being primarily reactive to primarily proactive, it is important for research results to be reported in a scientifically rigorous manner. Such reporting will allow for the standard pillars of science, namely replication, meta-analysis, and theory building. In this paper we aim to establish a baseline of the state of scientific work in security through the analysis of indicators of scientific research as reported in the papers from the 2015 IEEE Symposium on Security and Privacy. To conduct this analysis, we developed a series of rubrics to determine the completeness of the papers relative to the type of evaluation used (e.g. case study, experiment, proof). Our findings showed that while papers are generally easy to read, they often do not explicitly document some key information like the research objectives, the process for choosing the cases to include in the studies, and the threats to validity. We hope that this initial analysis will serve as a baseline against which we can measure the advancement of the science of security.}, booktitle={Proceedings of the Symposium and Bootcamp on the Science of Security}, publisher={ACM}, author={Carver, Jeffrey C. and Burcham, Morgan and Kocak, Sedef Akinli and Bener, Ayse and Felderer, Michael and Gander, Matthias and King, Jason and Markkula, Jouni and Oivo, Markku and Sauerwein, Clemens and et al.}, year={2016}, pages={38–51} }
 @article{riaz_king_slankas_williams_massacci_quesada-lopez_jenkins_2017, title={Identifying the implied: Findings from three differentiated replications on the use of security requirements templates}, volume={22}, ISSN={["1573-7616"]}, url={https://doi.org/10.1007/s10664-016-9481-1}, DOI={10.1007/s10664-016-9481-1}, abstractNote={Identifying security requirements early on can lay the foundation for secure software development. Security requirements are often implied by existing functional requirements but are mostly left unspecified. The Security Discoverer (SD) process automatically identifies security implications of individual requirements sentences and suggests applicable security requirements templates. The objective of this research is to support requirements analysts in identifying security requirements by automating the suggestion of security requirements templates that are implied by existing functional requirements. We conducted a controlled experiment in a graduate-level security class at North Carolina State University (NCSU) to evaluate the SD process in eliciting implied security requirements in 2014. We have subsequently conducted three differentiated replications to evaluate the generalizability and applicability of the initial findings. The replications were conducted across three countries at the University of Trento, NCSU, and the University of Costa Rica. We evaluated the responses of the 205 total participants in terms of quality, coverage, relevance and efficiency. We also develop shared insights regarding the impact of context factors such as time, motivation and support, on the study outcomes and provide lessons learned in conducting the replications. Treatment group, using the SD process, performed significantly better than the control group (at p-value <0.05) in terms of the coverage of the identified security requirements and efficiency of the requirements elicitation process in two of the three replications, supporting the findings of the original study. Participants in the treatment group identified 84 % more security requirements in the oracle as compared to the control group on average. Overall, 80 % of the 111 participants in the treatment group were favorable towards the use of templates in identifying security requirements. Our qualitative findings indicate that participants may be able to differentiate between relevant and extraneous templates suggestions and be more inclined to fill in the templates with additional support. Security requirements templates capture the security knowledge of multiple experts and can support the security requirements elicitation process when automatically suggested, making the implied security requirements more evident. However, individual participants may still miss out on identifying a number of security requirements due to empirical constraints as well as potential limitations on knowledge and security expertise.}, number={4}, journal={EMPIRICAL SOFTWARE ENGINEERING}, author={Riaz, Maria and King, Jason and Slankas, John and Williams, Laurie and Massacci, Fabio and Quesada-Lopez, Christian and Jenkins, Marcelo}, year={2017}, month={Aug}, pages={2127–2178} }
 @inproceedings{heckman_king_2016, place={New York, NY, USA}, title={Teaching Software Engineering Skills in CS1.5: Incorporating Real-world Practices and Tools (Abstract Only)}, url={http://doi.acm.org/10.1145/2839509.2850562}, DOI={10.1145/2839509.2850562}, abstractNote={Students learn best in environments where they can meaningfully engage with materials that emulate real-world scenarios. Incorporating software engineering best practices and supporting tools in introductory courses provides students the opportunity to engage in course materials as a novice member of the profession. We support student engagement with industry tools to support software engineering best practices for tutorials, in-class labs, and programming projects. The goal of the research is to improve student learning, engagement in the course and profession, and retention through the use of software engineering practices and tools that introduce students to the software engineering profession. A prior study on the incorporation of in-class laboratories, supported with software engineering best practices, on linear data structures showed an increase in engagement, but did not show a difference on student learning when compared with active learning lectures. We are currently expanding the study by incorporating in-class laboratories across a full semester of a CS1.5 class at NC State University. The poster presents the preliminary results from Fall 2015.}, booktitle={Proceedings of the 47th ACM Technical Symposium on Computing Science Education}, publisher={ACM}, author={Heckman, Sarah and King, Jason}, year={2016}, pages={696–697} }
 @article{king_stallings_riaz_williams_2017, title={To log, or not to log: using heuristics to identify mandatory log events - a controlled experiment}, volume={22}, ISSN={["1573-7616"]}, url={https://doi.org/10.1007/s10664-016-9449-1}, DOI={10.1007/s10664-016-9449-1}, number={5}, journal={EMPIRICAL SOFTWARE ENGINEERING}, author={King, Jason and Stallings, Jon and Riaz, Maria and Williams, Laurie}, year={2017}, month={Oct}, pages={2684–2717} }
 @inproceedings{heckman_king_winters_2015, place={New York, NY, USA}, title={Automating Software Engineering Best Practices Using an Open Source Continuous Integration Framework (Abstract Only)}, url={http://doi.acm.org/10.1145/2676723.2691921}, DOI={10.1145/2676723.2691921}, abstractNote={Ideally, software engineering courses should adequately reflect real-world software development so that students obtain a better understanding and experience with practices and techniques used in industry. Our objective is to improve software engineering courses by incorporating best practices for automated software engineering and facilitating rapid feedback for students using an open source continuous integration framework for evaluating student software development. The open source Jenkins Continuous Integration Server is the core of our framework, which provides a consistent environment for building student projects, executing automated test cases, calculating code coverage, executing static analysis, and generating reports for students. By using continuous integration, a common tool in real-world software development, we can incorporate software engineering best practices, introduce students to continuous integration in practice, and provide formative feedback to students throughout the software development lifecycle. We found that 76% or more of students in each of the classes that deploy our framework reported that using Jenkins increased their productivity, and that 84% or more of students in each of the classes reported that using Jenkins increased their code quality.}, booktitle={Proceedings of the 46th ACM Technical Symposium on Computer Science Education}, publisher={ACM}, author={Heckman, Sarah and King, Jason and Winters, Michael}, year={2015}, pages={677–677} }
 @inproceedings{king_pandita_williams_2015, place={New York, NY, USA}, title={Enabling Forensics by Proposing Heuristics to Identify Mandatory Log Events}, url={http://doi.acm.org/10.1145/2746194.2746200}, DOI={10.1145/2746194.2746200}, abstractNote={Software engineers often implement logging mechanisms to debug software and diagnose faults. As modern software manages increasingly sensitive data, logging mechanisms also need to capture detailed traces of user activity to enable forensics and hold users accountable. Existing techniques for identifying what events to log are often subjective and produce inconsistent results. The objective of this study is to help software engineers strengthen forensic-ability and user accountability by 1) systematically identifying mandatory log events through processing of unconstrained natural language software artifacts; and 2) proposing empirically-derived heuristics to help determine whether an event must be logged. We systematically extract each verb and object being acted upon from natural language software artifacts for three open-source software systems. We extract 3,513 verb-object pairs from 2,128 total sentences studied. Two raters classify each verb-object pair as either a mandatory log event or not. Through grounded theory analysis of discussions to resolve disagreements between the two raters, we develop 12 heuristics to help determine whether a verb-object pair describes an action that must be logged. Our heuristics help resolve 882 (96%) of 919 disagreements between the two raters. In addition, our results demonstrate that the proposed heuristics facilitate classification of 3,372 (96%) of 3,513 extracted verb-object pairs as either mandatory log events or not.}, booktitle={Proceedings of the 2015 Symposium and Bootcamp on the Science of Security}, publisher={ACM}, author={King, Jason and Pandita, Rahul and Williams, Laurie}, year={2015}, pages={6:1–6:11} }
 @book{bosu_corley_carver_gander_king_kocak_markkula_2015, title={Security Literature Review Paper Analysis Rubric}, institution={Technical Report #SERG-2015-01}, author={Bosu, A and Corley, C and Carver, J and Gander, M and King, J and Kocak, S and Markkula, J}, year={2015} }
 @inproceedings{riaz_king_slankas_williams_2014, title={Hidden in plain sight: Automatically identifying security requirements from natural language artifacts}, DOI={10.1109/re.2014.6912260}, abstractNote={Natural language artifacts, such as requirements specifications, often explicitly state the security requirements for software systems. However, these artifacts may also imply additional security requirements that developers may overlook but should consider to strengthen the overall security of the system. The goal of this research is to aid requirements engineers in producing a more comprehensive and classified set of security requirements by (1) automatically identifying security-relevant sentences in natural language requirements artifacts, and (2) providing context-specific security requirements templates to help translate the security-relevant sentences into functional security requirements. Using machine learning techniques, we have developed a tool-assisted process that takes as input a set of natural language artifacts. Our process automatically identifies security-relevant sentences in the artifacts and classifies them according to the security objectives, either explicitly stated or implied by the sentences. We classified 10,963 sentences in six different documents from healthcare domain and extracted corresponding security objectives. Our manual analysis showed that 46% of the sentences were security-relevant. Of these, 28% explicitly mention security while 72% of the sentences are functional requirements with security implications. Using our tool, we correctly predict and classify 82% of the security objectives for all the sentences (precision). We identify 79% of all security objectives implied by the sentences within the documents (recall). Based on our analysis, we develop context-specific templates that can be instantiated into a set of functional security requirements by filling in key information from security-relevant sentences.}, booktitle={2014 ieee 22nd international requirements engineering conference (re)}, author={Riaz, M. and King, Jason and Slankas, J. and Williams, L.}, year={2014}, pages={183–192} }
 @inproceedings{king_williams_2014, place={New York, NY, USA}, title={Log Your CRUD: Design Principles for Software Logging Mechanisms}, url={http://doi.acm.org/10.1145/2600176.2600183}, DOI={10.1145/2600176.2600183}, abstractNote={According to a 2011 survey in healthcare, the most commonly reported breaches of protected health information involved employees snooping into medical records of friends and relatives. Logging mechanisms can provide a means for forensic analysis of user activity in software systems by proving that a user performed certain actions in the system. However, logging mechanisms often inconsistently capture user interactions with sensitive data, creating gaps in traces of user activity. Explicit design principles and systematic testing of logging mechanisms within the software development lifecycle may help strengthen the overall security of software. The objective of this research is to observe the current state of logging mechanisms by performing an exploratory case study in which we systematically evaluate logging mechanisms by supplementing the expected results of existing functional black-box test cases to include log output. We perform an exploratory case study of four open-source electronic health record (EHR) logging mechanisms: OpenEMR, OSCAR, Tolven eCHR, and WorldVistA. We supplement the expected results of 30 United States government-sanctioned test cases to include log output to track access of sensitive data. We then execute the test cases on each EHR system. Six of the 30 (20%) test cases failed on all four EHR systems because user interactions with sensitive data are not logged. We find that viewing protected data is often not logged by default, allowing unauthorized views of data to go undetected. Based on our results, we propose a set of principles that developers should consider when developing logging mechanisms to ensure the ability to capture adequate traces of user activity.}, booktitle={Proceedings of the 2014 Symposium and Bootcamp on the Science of Security}, publisher={ACM}, author={King, Jason and Williams, Laurie}, year={2014}, pages={5:1–5:10} }
 @inproceedings{riaz_slankas_king_williams_2014, place={New York, NY, USA}, title={Using Templates to Elicit Implied Security Requirements from Functional Requirements - a Controlled Experiment}, url={http://doi.acm.org/10.1145/2652524.2652532}, DOI={10.1145/2652524.2652532}, abstractNote={Context: Security requirements for software systems can be challenging to identify and are often overlooked during the requirements engineering process. Existing functional requirements of a system can imply the need for security requirements. Systems having similar security objectives (e.g., confidentiality) often also share security requirements that can be captured in the form of reusable templates and instantiated in the context of a system to specify security requirements.
 Goal: We seek to improve the security requirements elicitation process by automatically suggesting appropriate security requirement templates implied by existing functional requirements.
 Method: We conducted a controlled experiment involving 50 graduate students enrolled in a software security course to evaluate the use of automatically-suggested templates in eliciting implied security requirements. Participants were divided into treatment (automatically-suggested templates) and control groups (no templates provided).
 Results: Participants using our templates identified 42% of all the implied security requirements in the oracle as compared to the control group, which identified only 16% of the implied security requirements. Template usage increased the efficiency of security requirements identified per unit of time.
 Conclusion: Automatically-suggested templates helped participants (security non-experts) think about security implications for the software system and consider more security requirements than they would have otherwise. We found that participants need more incentive than just a participatory grade when completing the task. Further, we recommend to ensure task completeness, participants either need a step-driven (i.e., wizard) approach or progress indicators to identify remaining work.}, booktitle={Proceedings of the 8th ACM/IEEE International Symposium on Empirical Software Engineering and Measurement}, publisher={ACM}, author={Riaz, Maria and Slankas, John and King, Jason and Williams, Laurie}, year={2014}, pages={22:1–22:10} }
 @inproceedings{king_williams_2013, place={Berkeley, CA, USA}, title={Cataloging and Comparing Logging Mechanism Specifications for Electronic Health Record Systems}, url={http://dl.acm.org/citation.cfm?id=2696523.2696527}, booktitle={Proceedings of the 2013 USENIX Conference on Safety, Security, Privacy and Interoperability of Health Information Technologies}, publisher={USENIX Association}, author={King, Jason and Williams, Laurie}, year={2013}, pages={4–4} }
 @inproceedings{king_2013, place={Piscataway, NJ, USA}, title={Measuring the Forensic-ability of Audit Logs for Nonrepudiation}, url={http://dl.acm.org/citation.cfm?id=2486788.2487022}, booktitle={Proceedings of the 2013 International Conference on Software Engineering}, publisher={IEEE Press}, author={King, Jason}, year={2013}, pages={1419–1422} }
 @article{king_smith_williams_2012, place={Hershey, PA, USA}, title={Audit Mechanisms in Electronic Health Record Systems: Protected Health Information May Remain Vulnerable to Undetected Misuse}, volume={3}, url={http://services.igi-global.com/resolvedoi/resolve.aspx?doi=10.4018/jcmam.2012040102}, DOI={10.4018/jcmam.2012040102}, abstractNote={Inadequate audit mechanisms may result in undetected misuse of data in software-intensive systems. In the healthcare domain, electronic health record (EHR) systems should log the creating, reading, updating, or deleting of privacy-critical protected health information. The objective of this paper is to assess electronic health record audit mechanisms to determine the current degree of auditing for non-repudiation and to assess whether general audit guidelines adequately address non-repudiation. The authors analyzed the audit mechanisms of two open source EHR systems, OpenEMR and Tolven eCHR, and one proprietary EHR system. The authors base the qualitative assessment on a set of 16 general auditable events and 58 black-box test cases for specific auditable events. The authors find that OpenEMR satisfies 62.5% of the general criteria and passes 63.8% of the black-box test cases. Tolven eCHR and the proprietary EHR system each satisfy less than 19% of the general criteria and pass less than 11% of the black-box test cases.}, number={2}, publisher={IGI Global}, author={King, Jason and Smith, Ben and Williams, Laurie}, year={2012}, pages={23–42} }
 @inproceedings{king_smith_williams_2012, place={New York, NY, USA}, title={Modifying Without a Trace: General Audit Guidelines Are Inadequate for Open-source Electronic Health Record Audit Mechanisms}, url={http://doi.acm.org/10.1145/2110363.2110399}, DOI={10.1145/2110363.2110399}, abstractNote={Without adequate audit mechanisms, electronic health record (EHR) systems remain vulnerable to undetected misuse. Users could modify or delete protected health information without these actions being traceable. The objective of this paper is to assess electronic health record audit mechanisms to determine the current degree of auditing for non-repudiation and to assess whether general audit guidelines adequately address non-repudiation. We derived 16 general auditable event types that affect non-repudiation based upon four publications. We qualitatively assess three open-source EHR systems to determine if the systems log these 16 event types. We find that the systems log an average of 12.5% of these event types. We also generated 58 black-box test cases based on specific auditable events derived from Certification Commission for Health Information Technology criteria. We find that only 4.02% of these tests pass. Additionally, 20% of tests fail in all three EHR systems. As a result, actions including the modification of patient demographics and assignment of user privileges can be executed without a trace of the user performing the action. The ambiguous nature of general auditable events may explain the inadequacy of auditing for non-repudiation. EHR system developers should focus on specific auditable events for managing protected health information instead of general events derived from guidelines.}, booktitle={Proceedings of the 2Nd ACM SIGHIT International Health Informatics Symposium}, publisher={ACM}, author={King, Jason Tyler and Smith, Ben and Williams, Laurie}, year={2012}, pages={305–314} }
 @inproceedings{secure logging and auditing in electronic health records systems: what can we learn from the payment card industry_2012, place={Bellevue, WA}, url={https://www.usenix.org/conference/healthsec12/workshop-program/presentation/King}, booktitle={Presented as part of the 3rd USENIX Workshop on Health Security and Privacy}, publisher={USENIX}, year={2012} }
 @inproceedings{smith_austin_brown_king_lankford_meneely_williams_2010, place={New York, NY, USA}, title={Challenges for Protecting the Privacy of Health Information: Required Certification Can Leave Common Vulnerabilities Undetected}, url={http://doi.acm.org/10.1145/1866914.1866916}, DOI={10.1145/1866914.1866916}, abstractNote={The use of electronic health record (EHR) systems by medical professionals enables the electronic exchange of patient data, yielding cost and quality of care benefits. The United States American Recovery and Reinvestment Act (ARRA) of 2009 provides up to $34 billion for meaningful use of certified EHR systems. But, will these certified EHR systems provide the infrastructure for secure patient data exchange? As a window into the ability of current and emerging certification criteria to expose security vulnerabilities, we performed exploratory security analysis on a proprietary and an open source EHR. We were able to exploit a range of common code-level and design-level vulnerabilities. These common vulnerabilities would have remained undetected by the 2011 security certification test scripts from the Certification Commission for Health Information Technology, the most widely used certification process for EHR systems. The consequences of these exploits included, but were not limited to: exposing all users' login information, the ability of any user to view or edit health records for any patient, and creating a denial of service for all users. Based upon our results, we suggest that an enhanced set of security test scripts be used as entry criteria to the EHR certification process. Before certification bodies spend the time to certify that an EHR application is functionally complete, they should have confidence that the software system meets a basic level of security competence.}, booktitle={Proceedings of the Second Annual Workshop on Security and Privacy in Medical and Home-care Systems}, publisher={ACM}, author={Smith, Ben and Austin, Andrew and Brown, Matt and King, Jason T. and Lankford, Jerrod and Meneely, Andrew and Williams, Laurie}, year={2010}, pages={1–12} }