@article{kokotovich_barnhill-dilling_elsensohn_li_delborne_burrack_2022, title={Stakeholder engagement to inform the risk assessment and governance of gene drive technology to manage spotted-wing drosophila}, volume={307}, ISSN={["1095-8630"]}, url={https://doi.org/10.1016/j.jenvman.2022.114480}, DOI={10.1016/j.jenvman.2022.114480}, abstractNote={Emerging biotechnologies, such as gene drive technology, are increasingly being proposed to manage a variety of pests and invasive species. As one method of genetic biocontrol, gene drive technology is currently being developed to manage the invasive agricultural pest spotted-wing drosophila (Drosophila suzukii, SWD). While there have been calls for stakeholder engagement on gene drive technology, there has been a lack of empirical work, especially concerning stakeholder engagement to inform risk assessment. To help address this gap and inform future risk assessments and governance decisions for SWD gene drive technology, we conducted a survey of 184 SWD stakeholders to explore how they define and prioritize potential benefits and potential adverse effects from proposed SWD gene drive technology. We found that stakeholders considered the most important potential benefits of SWD gene drive technology to be: 1) Decrease in the quantity or toxicity of pesticides used, and 2) Decrease in SWD populations. Stakeholders were most concerned about the potential adverse effects of: 1) Decrease in beneficial insects, 2) Increase in non-SWD secondary pest infestations, and 3) Decrease in grower profits. Notably, we found that even stakeholders who expressed support for the use of SWD gene drive technology expressed concerns about potential adverse effects from the technology, emphasizing the need to move past simplistic, dichotomous views of what it means to support or oppose a technology. These findings suggest that instead of focusing on the binary question of whether stakeholders support or oppose SWD gene drive technology, it is more important to identify and assess the factors that are consequential to stakeholder decision making - including, for example, exploring whether and under what conditions key potential adverse effects and potential benefits would result from the use of SWD gene drive technology.}, journal={JOURNAL OF ENVIRONMENTAL MANAGEMENT}, publisher={Elsevier BV}, author={Kokotovich, Adam E. and Barnhill-Dilling, S. Kathleen and Elsensohn, Johanna E. and Li, Richard and Delborne, Jason A. and Burrack, Hannah}, year={2022}, month={Apr} }
 @article{cummings_kuzma_kokotovich_glas_grieger_2021, title={Barriers to responsible innovation of nanotechnology applications in food and agriculture: A study of US experts and developers}, volume={23}, ISSN={["2452-0748"]}, DOI={10.1016/j.impact.2021.100326}, abstractNote={The use of nanotechnology and engineered nanomaterials in food and agriculture (nano-agrifood) sectors is intended to provide several potential benefits to consumers and society, such as the provision of more nutritious processed foods, edible food coatings to extend shelf lives of fresh cut produce, and more sustainable alternatives to traditional agrochemicals. The responsible innovation of nano-agrifoods may be particularly important to pursue given previous case studies involving other agrifood technologies that experienced significant public consternation. Here, we define responsible innovation following Stilgoej et al. (2013) that establishes processes to iteratively review and reflect upon one's innovation, engage stakeholders in dialogue, and to be open and transparent throughout innovation stages - processes that go beyond primary focuses of understanding environmental, health, and safety impacts of nano-enabled products and implementing safe-by-design principles. Despite calls for responsible nano-innovation across diverse sectors, it has not yet been clear what types of barriers are faced by nano-agrifood researchers and innovators in particular. This study therefore identifies and builds the first typology of barriers to responsible innovation as perceived by researchers and product developers working in nano-agrifood sectors in the United States. Our findings report 5 key barriers to responsible innovation of nano-agrifoods: Lack of Data (reported by 70% of all interview participants, and represented 34.6% of all barrier-related excerpts), Lack of Product Oversight (reported by 60% of participants, and represented 28.7% of excerpts), Need for Ensuring Marketability & Use (reported by 70% of participants, and represented 21.3% of all barrier-related excerpts), Need for Increased Collaboration (reported by 40% of participants, and represented 10.3% of excerpts), and finally Lack of Adequate Training & Workforce (reported by 30% of participants, and represented by 5.1% of excerpts). We also relate these key barriers across three main nano-innovation phases, including 1) Scientific and Technical R&D, 2) Product Oversight, and 3) Post-commercialization Marketability & Use, and discuss how these barriers may impact stakeholders as well as present opportunities to align with principles of responsible innovation. Overall, these findings may help illuminate challenges that researchers and innovators face in the pursuit of responsible innovation relevant for the field of nanotechnology with relevancy for other emerging food and agricultural technologies more broadly.}, journal={NANOIMPACT}, author={Cummings, Christopher L. and Kuzma, Jennifer and Kokotovich, Adam and Glas, David and Grieger, Khara}, year={2021}, month={Jul} }
 @article{kokotovich_kuzma_cummings_grieger_2021, title={Responsible Innovation Definitions, Practices, and Motivations from Nanotechnology Researchers in Food and Agriculture}, volume={15}, ISSN={1871-4757 1871-4765}, url={http://dx.doi.org/10.1007/s11569-021-00404-9}, DOI={10.1007/s11569-021-00404-9}, abstractNote={Abstract The growth of responsible innovation (RI) scholarship has been mirrored by a proliferation of RI definitions and practices, as well as a recognition of the importance of context for RI. This study investigates how researchers in the field of nanotechnology for food and agriculture (nano-agrifoods) define and practice RI, as well as what motivations they see for pursuing RI. We conducted 20 semi-structured interviews with nano-agrifood researchers from industry and academia in the USA, where we asked them to describe their RI definitions, practices, and motivations. We analyzed the emergent themes from these interviews, including how the themes aligned with four prominent RI principles (anticipation, inclusion, reflexivity, responsiveness). We found that nano-agrifood researchers largely focused their descriptions of RI definitions, practices, and motivations around a narrow envisioning of the RI principle of anticipation — emphasizing product safety, efficacy, and efficiency. We also found noteworthy tensions surrounding the less frequently mentioned RI principles. For example, some researchers envisioned inclusion as a way to align products with industry interests while others saw it as a way to align products with the public good. Concerning motivations for RI, some researchers viewed RI as a way to protect one’s reputation and avoid lawsuits while others viewed it as a way to improve human well-being and solve societal problems. Given these findings, future efforts to foster RI within nano-agrifoods should promote discussions among researchers concerning what it means to responsibly innovate and what practices this could entail, particularly beyond ensuring product safety, efficacy, and efficiency.}, number={3}, journal={NanoEthics}, publisher={Springer Science and Business Media LLC}, author={Kokotovich, Adam E. and Kuzma, Jennifer and Cummings, Christopher L. and Grieger, Khara}, year={2021}, month={Dec}, pages={229–243} }
 @article{grieger_merck_cuchiara_binder_kokotovich_cummings_kuzma_2021, title={Responsible innovation of nano-agrifoods: Insights and views from U.S. stakeholders}, volume={24}, ISSN={2452-0748}, url={http://dx.doi.org/10.1016/j.impact.2021.100365}, DOI={10.1016/j.impact.2021.100365}, abstractNote={To date, there has been little published work that has elicited diverse stakeholder views of nano-agrifoods and of how nano-agrifoods align with the goals of responsible innovation. This paper aims to fill this research gap by investigating views of nano-agrifoods, how well their development adheres to principles of responsible innovation, and potential challenges for achieving responsible nano-agrifood innovation. Using an online engagement platform, we find that U.S. stakeholder views of responsible innovation were dominated by environmental, health, and safety (EHS) contexts, considerations of societal impacts, opportunities for stakeholder engagement, and responding to societal needs. These views overlap with scholarly definitions of responsible innovation, albeit stakeholders were more focused on impacts of products, while the field of responsible innovation strives for more “upstream” considerations of the process of innovation. We also find that views of nano-agrifoods differed across applications with dietary supplements and improved whitening of infant formula viewed least favorably, and environmental health or food safety applications viewed most favorably. These findings align with the larger body of literature, whereby stakeholders are expected to be more supportive of nanotechnology used in agricultural applications compared to directly within food and food supplements. Overall, participants indicated they held relatively neutral views on research and innovation for nano-agrifoods being conducted responsibly, and they identified key challenges to ensuring their responsible innovation that were related to uncertainties in EHS studies, the need for public understanding and acceptance, and adequate regulation. In light of these results, we recommend future research efforts on EHS impacts and risk-benefit frameworks for nano-agrifoods, better understanding stakeholder views on what constitutes effective regulation, and addressing challenges with effective regulation and responsible innovation practices.}, journal={NanoImpact}, publisher={Elsevier BV}, author={Grieger, Khara D. and Merck, Ashton W. and Cuchiara, Maude and Binder, Andrew R. and Kokotovich, Adam and Cummings, Christopher L. and Kuzma, Jennifer}, year={2021}, month={Oct}, pages={100365} }
 @article{hartley_smith_kokotovich_opesen_habtewold_ledingham_raymond_rwabukwali_2021, title={Ugandan stakeholder hopes and concerns about gene drive mosquitoes for malaria control: new directions for gene drive risk governance}, volume={20}, ISSN={["1475-2875"]}, DOI={10.1186/s12936-021-03682-6}, abstractNote={Abstract Background The African Union’s High-Level Panel on Emerging Technologies identified gene drive mosquitoes as a priority technology for malaria elimination. The first field trials are expected in 5–10 years in Uganda, Mali or Burkina Faso. In preparation, regional and international actors are developing risk governance guidelines which will delineate the framework for identifying and evaluating risks. Scientists and bioethicists have called for African stakeholder involvement in these developments, arguing the knowledge and perspectives of those people living in malaria-afflicted countries is currently missing. However, few African stakeholders have been involved to date, leaving a knowledge gap about the local social-cultural as well as ecological context in which gene drive mosquitoes will be tested and deployed. This study investigates and analyses Ugandan stakeholders’ hopes and concerns about gene drive mosquitoes for malaria control and explores the new directions needed for risk governance. Methods This qualitative study draws on 19 in-depth semi-structured interviews with Ugandan stakeholders in 2019. It explores their hopes for the technology and the risks they believed pertinent. Coding began at a workshop and continued through thematic analysis. Results Participants’ hopes and concerns for gene drive mosquitoes to address malaria fell into three themes: (1) ability of gene drive mosquitoes to prevent malaria infection; (2) impacts of gene drive testing and deployment; and, (3) governance. Stakeholder hopes fell almost exclusively into the first theme while concerns were spread across all three. The study demonstrates that local stakeholders are able and willing to contribute relevant and important knowledge to the development of risk frameworks. Conclusions International processes can provide high-level guidelines, but risk decision-making must be grounded in the local context if it is to be robust, meaningful and legitimate. Decisions about whether or not to release gene drive mosquitoes as part of a malaria control programme will need to consider the assessment of both the risks and the benefits of gene drive mosquitoes within a particular social, political, ecological, and technological context. Just as with risks, benefits—and importantly, the conditions that are necessary to realize them—must be identified and debated in Uganda and its neighbouring countries.}, number={1}, journal={MALARIA JOURNAL}, author={Hartley, Sarah and Smith, Robert D. J. and Kokotovich, Adam and Opesen, Chris and Habtewold, Tibebu and Ledingham, Katie and Raymond, Ben and Rwabukwali, Charles B.}, year={2021}, month={Mar} }
 @article{kokotovich_delborne_elsensohn_burrack_2020, title={Emerging Technologies for Invasive Insects: The Role of Engagement}, volume={113}, ISSN={0013-8746 1938-2901}, url={http://dx.doi.org/10.1093/aesa/saz064}, DOI={10.1093/aesa/saz064}, abstractNote={Abstract Emerging technologies have the potential to offer new applications for managing invasive insects. While scientific and technological advancements are vital to realizing this potential, the successful development and use of these applications will also largely depend on community and stakeholder engagement. To contribute to a relevant and rigorous envisioning of engagement for emerging technologies for invasive insects (ETII), we begin by reviewing key insights on engagement from three scholarly fields: invasive species management, responsible research and innovation, and ecological risk assessment. Across these fields we glean best practices for engagement for ETII: 1) pursue engagement across decision phases and sectors; 2) select context-appropriate participants and methods; and 3) recognize and navigate engagement-related tensions. We illustrate these best practices by describing an ongoing project that uses engagement to inform risk assessment and broader decision making on biotechnologies being developed to address the Spotted-wing Drosophila (Drosophila suzukii) invasive fruit fly. We describe completed and planned engagement activities designed to identify and prioritize potential adverse effects, benefits, management actions, and research actions of the proposed genetically engineered sterile male, gene drive, and RNAi biotechnologies. In the face of broadening calls for engagement on emerging technologies, this article provides theoretical and empirical insights that can guide future engagement for ETII.}, number={4}, journal={Annals of the Entomological Society of America}, publisher={Oxford University Press (OUP)}, author={Kokotovich, Adam E and Delborne, Jason A and Elsensohn, Johanna and Burrack, Hannah}, editor={Morisette, JeffreyEditor}, year={2020}, month={Mar}, pages={266–279} }
 @inbook{hartley_kokotovich_2018, title={Disentangling risk assessment}, ISBN={9781526106476}, url={http://dx.doi.org/10.7765/9781526106476.00019}, DOI={10.7765/9781526106476.00019}, abstractNote={European and North American regulatory agencies have a statutory obligation to involve the public in risk decision-making and in recent years many have ‘opened up’ these traditionally scientific domains to public input through on-line consultations. However, these statutory obligations are not met in practice and opportunities for public involvement are not exploited to their full potential. We argue this failure is due to a considerable lack of clarity in the literature and in practice about which publics should be involved in risk assessment and at what point they should participate. To remedy this situation, we draw on theoretical, empirical and prescriptive literatures to disentangle risk assessment. First, we find that effective and legitimate public involvement is dependent upon the degree to which value-judgements are acknowledged in the different components of risk assessment. Second, we explore variations in the prescription literatures of the United States National Research Council and the Codex Alimentarius Commission. Third, we examine the way in which risk assessment is disentangled in practice through the case study of the European Food Safety Authority. Finally, we draw on these findings to reassemble public involvement in risk assessment, making clear who should be involved, where and, importantly, why.}, booktitle={Science and the politics of openness}, publisher={Manchester University Press}, author={Hartley, Sarah and Kokotovich, Adam}, year={2018}, month={Feb} }
 @inbook{hartley_kokotovich_2018, place={Manchester, England}, title={Disentangling risk assessment: new roles for experts and publics}, url={http://www.oapen.org/download?type=document&docid=643155}, booktitle={Science and the Politics of Openness: Here Be Monsters}, publisher={Manchester University Press}, author={Hartley, S. and Kokotovich, A.}, editor={Nerlich, B. and Hartley, S. and Raman, S. and Smith, A.Editors}, year={2018}, pages={176–194} }
 @article{escobar_mallez_mccartney_lee_zielinski_ghosal_bajer_wagner_nash_tomamichel_et al._2017, title={Aquatic Invasive Species in the Great Lakes Region: An Overview}, volume={26}, ISSN={2330-8249 2330-8257}, url={http://dx.doi.org/10.1080/23308249.2017.1363715}, DOI={10.1080/23308249.2017.1363715}, abstractNote={Aquatic invasive species (AIS) are of concern in North America due to their devastating impacts on ecosystems and economies. The Great Lakes region is particularly vulnerable to AIS introduction an...}, number={1}, journal={Reviews in Fisheries Science & Aquaculture}, publisher={Informa UK Limited}, author={Escobar, Luis E. and Mallez, Sophie and McCartney, Michael and Lee, Christine and Zielinski, Daniel P. and Ghosal, Ratna and Bajer, Przemyslaw G. and Wagner, Carli and Nash, Becca and Tomamichel, Megan and et al.}, year={2017}, month={Sep}, pages={121–138} }
 @article{kokotovich_andow_2017, title={Exploring tensions and conflicts in invasive species management: The case of Asian carp}, volume={69}, ISSN={1462-9011}, url={http://dx.doi.org/10.1016/j.envsci.2016.12.016}, DOI={10.1016/j.envsci.2016.12.016}, abstractNote={There is a growing recognition that scientific and social conflict pervades invasive species management, but there is a need for empirical work that can help better understand these conflicts and how they can be addressed. We examined the tensions and conflicts facing invasive Asian carp management in Minnesota by conducting 16 in-depth interviews with state and federal agency officials, academics, and stakeholders. Interviewees discussed the tensions and conflicts they saw impacting management, their implications, and what could be done to address them. We found three key areas of conflict and tension in Asian carp management: 1) scientific uncertainty concerning the impacts of Asian carp and the efficacy and non-target effects of possible management actions; 2) social uncertainty concerning both the lack of societal agreement on how to respond to Asian carp and the need to avoid acting from apathy and/or fear; and 3) the desired approach to research and management – whether it is informed by “political need” or “biological reality”. Our study of these tensions and conflicts reveals their importance to Asian carp management and to invasive species management, more broadly. We conclude with a discussion of possible ways to address these areas of tension and conflict, including the potential of deliberative, participatory approaches to risk-related decision making and the need to productively engage with apathy and fear.}, journal={Environmental Science & Policy}, publisher={Elsevier BV}, author={Kokotovich, Adam E. and Andow, David A.}, year={2017}, month={Mar}, pages={105–112} }
 @article{kuzma_kokotovich_kuzhabekova_2016, title={Attitudes Towards Governance of Gene Editing}, volume={18}, number={1}, journal={Asian Biotechnology and Development Review}, author={Kuzma, J. and Kokotovich, A. and Kuzhabekova, A.}, year={2016}, pages={69–92} }
 @article{kokotovich_kuzma_2014, title={Conflicting Futures}, volume={34}, ISSN={0270-4676 1552-4183}, url={http://dx.doi.org/10.1177/0270467614565695}, DOI={10.1177/0270467614565695}, abstractNote={Novel targeted genetic modification (TagMo) techniques for plants have the potential to increase the speed and ease of genetic modification and fall outside existing regulatory authority. We conducted 31 interviews with expert-stakeholders to explore the differing visions they have for the future of plant TagMo environmental regulation. To guide our analysis we review the tenets of anticipatory governance in light of future studies literature on emerging technology, focusing on how to contribute to reflexivity by making explicit the assumptions within envisioned futures. Our findings reveal that the environmental regulation futures articulated by expert-stakeholders could be classified into three categories—optimistic, pragmatic, and critical—based on their differing underlying assumptions concerning what constitutes environmental risk and the adequacy of existing U.S. genetically modified plant regulations. By gathering these diverse perspectives on the future and studying how they differ, we hope to further the anticipatory governance-informed engagement with regulation and foster a more productive discussion of plant TagMo regulation.}, number={3-4}, journal={Bulletin of Science, Technology & Society}, publisher={SAGE Publications}, author={Kokotovich, Adam and Kuzma, Jennifer}, year={2014}, month={Jun}, pages={108–120} }
 @article{kokotovich_zeilinger_2011, title={Exploring social and institutional variation across oak wilt risk management programs in Minnesota, USA}, volume={10}, ISSN={1618-8667}, url={http://dx.doi.org/10.1016/j.ufug.2010.09.004}, DOI={10.1016/j.ufug.2010.09.004}, abstractNote={The Minnesota Department of Natural Resources, in cooperation with United States Forest Service, has implemented a risk management program to fund local government action aimed at containing the exotic invasive oak tree pathogen, Ceratocystis fagacearum – the causal agent of oak wilt. In administering the oak wilt ReLeaf program, the Minnesota Department of Natural Resources and United States Forest Service sought to facilitate efficient and effective management through enabling and mobilizing local land managers. Because the program is decentralized and because cities and counties are heterogeneous, local land managers take advantage of the program in different ways. Our goal in this study was to document the social and institutional variation in oak wilt management programs at the local (city and county) level, and provide preliminary insights into the causes of the variation. Specifically we asked: what factors constrain and facilitate oak wilt management at the local level; and how do ReLeaf program funds influence local management? To address these questions, we completed 12 in-depth qualitative interviews with foresters and private contractors, yielding information on 16 oak wilt management programs in the Minneapolis/St. Paul metropolitan area, Minnesota, USA. Through our analysis we identified five emergent themes that produce and reflect the overall variation in oak wilt management programs: (1) existence and enforcement of Potential Spore Producing Tree removal ordinances; (2) amount of local resources and potential effect of decrease in state funding; (3) differences across urban and rural areas; (4) differences in program goals and landowner cost sharing structures; and (5) ability to attain the Minnesota Department of Natural Resources’ suppression goal. These findings describe factors contributing to variation across oak wilt management programs, illustrate challenges facing local invasive species management efforts, and provide insights for designing improved invasive species management programs at the federal, state and local level.}, number={1}, journal={Urban Forestry & Urban Greening}, publisher={Elsevier BV}, author={Kokotovich, Adam E. and Zeilinger, Adam R.}, year={2011}, month={Jan}, pages={39–45} }
 @article{kuzma_kokotovich_2011, title={Renegotiating GM crop regulation}, volume={12}, ISSN={1469-221X 1469-3178}, url={http://dx.doi.org/10.1038/embor.2011.160}, DOI={10.1038/embor.2011.160}, abstractNote={Outlook12 August 2011free access Renegotiating GM crop regulation Targeted gene-modification technology raises new issues for the oversight of genetically modified crops Jennifer Kuzma Jennifer Kuzma Humphrey School of Public Affairs and Center for Science, University of Minnesota, Minneapolis, USA Search for more papers by this author Adam Kokotovich Adam Kokotovich Center for Science, Technology & Public Policy, University of Minnesota, St Paul, USA Search for more papers by this author Jennifer Kuzma Jennifer Kuzma Humphrey School of Public Affairs and Center for Science, University of Minnesota, Minneapolis, USA Search for more papers by this author Adam Kokotovich Adam Kokotovich Center for Science, Technology & Public Policy, University of Minnesota, St Paul, USA Search for more papers by this author Author Information Jennifer Kuzma1 and Adam Kokotovich2 1Humphrey School of Public Affairs and Center for Science, University of Minnesota, Minneapolis, USA 2Center for Science, Technology & Public Policy, University of Minnesota, St Paul, USA EMBO Reports (2011)12:883-888https://doi.org/10.1038/embor.2011.160 PDFDownload PDF of article text and main figures. ToolsAdd to favoritesDownload CitationsTrack CitationsPermissions ShareFacebookTwitterLinked InMendeleyWechatReddit Figures & Info In 2010, more than 85% of the corn acreage and more than 90% of the soybean acreage in the USA was planted with genetically modified (GM) crops (USDA, 2010). Most of those crops contained transgenes from other species, such as bacteria, that confer resistance to herbicides or tolerance to insect pests, and that were introduced into plant cells using Agrobacterium or other delivery methods. The resulting ‘transformed’ cells were regenerated into GM plants that were tested for the appropriate expression of the transgenes, as well as for whether the crop posed an unacceptable environmental or health risk, before being approved for commercial use. The scientific advances that enabled the generation of these GM plants took place in the early 1980s and have changed agriculture irrevocably, as evidenced by the widespread adoption of GM technology. They have also triggered intense debates about the potential risks of GM crops for human health and the environment and new forms of regulation that are needed to mitigate this. There is also continued public resistance to GM crops, particularly in Europe. Plant genetic engineering is at a technological inflection point Plant genetic engineering is at a technological inflection point. New technologies enable more precise and subtler modification of plant genomes (Weinthal et al, 2010) than the comparably crude methods that were used to create the current stock of GM crops (Fig 1A). These methods allow scientists to insert foreign DNA into the plant genome at precise locations, remove unwanted DNA sequences or introduce subtle modifications, such as single-base substitutions that alter the activity of individual genes. They also raise serious questions about the regulation of GM crops: how do these methods differ from existing techniques and how will the resulting products be regulated? Owing to the specificity of these methods, will resulting products fall outside existing definitions of GM crops and, as a result, be regulated similarly to conventional crops? How will the definition and regulation of GM crops be renegotiated and delineated in light of these new methods? Figure 1.Comparing traditional transgenesis, targeted transgenesis, targeted mutagenesis and gene replacement. (A) In traditional transgenesis, genes introduced into plant cells integrate at random chromosomal positions. This is illustrated here for a bacterial gene that confers herbicide resistance (Herbr). The plant encodes a gene for the same enzyme, however due to DNA-sequence differences between the bacterial and plant forms of the gene, the plant gene does not confer herbicide resistance (Herbs). (B) The bacterial herbicide-resistance gene can be targeted to a specific chromosomal location through the use of engineered nucleases. The nucleases recognize a specific DNA sequence and create a chromosome break. The bacterial gene is flanked by sequences homologous to the target site and recombines with the plant chromosome at the break site, resulting in a targeted insertion. (C) Engineered nucleases can be used to create targeted gene knockouts. In this illustration, a second nuclease recognizes the coding sequence of the Herbs gene. Cleavage and repair in the absence of a homologous template creates a mutation (orange). (D) A homologous DNA donor can be used to repair targeted breaks in the Herbs gene. This allows sequence changes to be introduced into the native plant gene that confer herbicide resistance. Only a single base change is needed in some instances. Download figure Download PowerPoint Of the new wave of targeted genetic modification (TagMo) techniques, one of the most thoroughly developed is TagMo, which uses engineered zinc-finger nucleases or meganucleases to create DNA double-stranded breaks at specific genomic locations (Townsend et al, 2009; Shukla et al, 2009; Gao et al, 2010). This activates DNA repair mechanisms, which genetic engineers can use to alter the target gene. If, for instance, a DNA fragment is provided that has sequence similarity with the site at which the chromosome is broken, the repair mechanism will use this fragment as a template for repair through homologous recombination (Fig 1B). In this way, any DNA sequence, for instance a bacterial gene that confers herbicide resistance, can be inserted at the site of the chromosome break. TagMos can also be used without a repair template to make single-nucleotide changes. In this case, the broken chromosomes are rejoined imprecisely, creating small insertions or deletions at the break site (Fig 1C) that can alter or knock out gene function. TagMo technology would, therefore, challenge regulatory policies both in the USA and, even more so, in the [EU]… The greatest potential of TagMo technology is in its ability to modify native plant genes in directed and targeted ways. For example, the most widely used herbicide-resistance gene in GM crops comes from bacteria. Plants encode the same enzyme, but it does not confer herbicide resistance because the DNA sequence is different. Yet, resistant forms of the plant gene have been identified that differ from native genes by only a few nucleotides. TagMo could therefore be used to transfer these genes from a related species into a crop to replace the existing genes (Fig 1D) or to exchange specific nucleotides until the desired effect is achieved. In either case, the genetic modification would not necessarily involve transfer of DNA from another species. TagMo technology would, therefore, challenge regulatory policies both in the USA and, even more so, in the European Union (EU). TagMo enables more sophisticated modifications of plant genomes that, in some cases, could be achieved by classical breeding or mutagenesis, which are not formally regulated. On the other hand, TagMo might also be used to introduce foreign genes without using traditional recombinant DNA techniques. As a result, TagMo might fall outside of existing US and EU regulatory definitions and scrutiny. In the USA, federal policies to regulate GM crops could provide a framework in which to consider the way TagMo-derived crops might be regulated (Fig 2; Kuzma & Meghani, 2009; Kuzma et al, 2009; Thompson, 2007; McHughen & Smyth, 2008). In 1986, the Office of Science and Technology Policy established the Coordinated Framework for the Regulation of Biotechnology (CFRB) to oversee the environmental release of GM crops and their products (Office of Science and Technology Policy, 1986). The CFRB involves many federal agencies and is still in operation today (Kuzma et al, 2009). It was predicated on the views that regulation should be based on science and that the risks posed by GM crops were the “same in kind” as those of non-GM products; therefore no new laws were deemed to be required (National Research Council, 2000). Figure 2.Brief history of the regulation of genetic engineering (Kuzma et al, 2009). EPA, Environmental Protection Agency; FIFRA, Federal Insecticide, Fungicide and Rodenticide Act; FDA, Food and Drug Administration; FPPA, Farmland Protection Policy Act; GMO, genetically modified organism; TOSCA, Toxic Substances Control Act; USDA, United States Department of Agriculture. Download figure Download PowerPoint Various old and existing statutes were interpreted somewhat loosely in order to oversee the regulation of GM plants. Depending on the nature of the product, one or several federal agencies might be responsible. GM plants can be regulated by the US Department of Agriculture (USDA) under the Federal Plant Pest Act as ‘plant pests’ if there is a perceived threat of them becoming ‘pests’ (similarly to weeds). Alternatively, if they are pest-resistant, they can be interpreted as ‘plant pesticides’ by the US Environmental Protection Agency (EPA) under the Federal Insecticide, Fungicide, and Rodenticide Act. Each statute requires some kind of pre-market or pre-release biosafety review—evaluation of potential impacts on other organisms in the environment, gene flow between the GM plant and wild relatives, and potential adverse effects on ecosystems. By contrast, the US Food and Drug Administration (FDA) treats GM food crops as equivalent to conventional food products; as such, no special regulations were promulgated under the Federal Food Drug and Cosmetic Act for GM foods. The agency established a pre-market consultation process for GM and other novel foods that is entirely voluntary. …TagMo-derived crops come in several categories relevant to regulation… Finally, and important for our discussion, the US oversight system was built mostly around the idea that GM plants should be regulated on the basis of characteristics of the end-product and not on the process that is used to create them. In reality, however, the process used to create crops is significant, which is highlighted by the fact that the USDA uses a process-based regulatory trigger (McHughen & Smyth, 2008). Instead of being inconsequential, it is important for oversight whether a plant is considered to be a result of GM. How will crops created by TagMo fit into this regulatory framework? If only subtle changes were made to individual genes, the argument could be made that the products are analogous to mutated conventional crops, which are neither regulated nor subject to pre-market or pre-release biosafety assessments (Breyer et al, 2009). However, single mutations are not without risks; for example, they can lead to an increase in expressed plant toxins (National Research Council, 1989, 2000, 2002, 2004; Magana-Gomez & de la Barca 2009). Conversely, if new or foreign genes are introduced through TagMo methods, the resulting plants might not differ substantially from existing GM crops. Thus, TagMo-derived crops come in several categories relevant to regulation: TagMo-derived crops with inserted foreign DNA from sexually compatible or incompatible species; TagMo-derived crops with no DNA inserted, for instance those in which parts of the chromosome have been deleted or genes inactivated; and TagMo-derived crops that either replace a gene with a modified version or change its nucleotide sequence (Fig 1). TagMo-derived crops with foreign genetic material inserted are most similar to traditional GM crops, according to the USDA rule on “Importation, Interstate Movement, and Release Into the Environment of Certain Genetically Engineered Organisms”, which defines genetic engineering as “the genetic modification of organisms by recombinant DNA (rDNA) techniques” (USDA, 1997). In contrast to conventional transgenesis, TagMo enables scientists to predefine the sites into which foreign genes are inserted. If the site of foreign DNA insertion has been previously characterized and shown to have no negative consequences for the plant or its products, then perhaps regulatory requirements to characterize the insertion site and its effects on the plant could be streamlined. TagMo might be used to introduce foreign DNA from sexually compatible or incompatible species into a host organism, either by insertion or replacement. For example, foreign DNA from one species of Brassica—mustard family—can be introduced into another species of Brassica. Alternatively, TagMo might be used to introduce foreign DNA from any organism into the host, such as from bacteria or animals into plants. Arguments have been put forth advocating less stringent regulation of GM crops with cisgenic DNA sequences that come from sexually compatible species (Schouten et al, 2006). Russell and Sparrow (2008) critically evaluate these arguments and conclude that cisgenic GM crops may still have novel traits in novel settings and thus give rise to novel hazards. Furthermore, if cisgenics are not regulated, it might trigger a public backlash, which could be more costly in the long run (Russell & Sparrow, 2008). TagMo-derived crops with genetic sequences from sexually compatible species should therefore still be considered for regulation. Additional clarity and consistency is needed with respect to how cisgenics are defined in US regulatory policy, regardless of whether they are generated by established methods or by TagMo. The USDA regulatory definition of a GM crops is vague, and the EPA has a broad categorical exemption in its rules for GM crops with sequences from sexually compatible species (EPA, 2001). Public failures will probably ensue if TagMo crops slip into the market under the radar without adequate oversight The deletion of DNA sequences by TagMo to knock out a target gene is potentially of great agronomic value, as it could remove undesirable traits. For instance, it could eliminate anti-nutrients such as trypsin inhibitors in soybean that prevent the use of soy proteins by animals, or compounds that limit the value of a crop as an industrial material, such as ricin, which contaminates castor oil. Many mutagenesis methods yield similar products as TagMos. However, most conventional mutagenesis methods, including DNA alkylating agents or radioactivity, provide no precision in terms of the DNA sequences modified, and probably cause considerable collateral damage to the genome. It could be argued that TagMo is less likely to cause unpredicted genomic changes; however, additional research is required to better understand off-target effects—that is, unintended modification of other sites—by various TagMo platforms. We propose that the discussion about how to regulate TagMo crops should be open, use public engagement and respect several criteria of oversight Generating targeted gene knockouts (Fig 1C) does not directly involve transfer of foreign DNA, and such plants might seem to warrant an unregulated status. However, most TagMos use reagents such as engineered nucleases, which are created by rDNA methods. The resulting product might therefore be classified as a GM crop under the existing USDA definition for genetic engineering (USDA, 1997) since most TagMos are created by introducing a target-specific nuclease gene into plant cells. It is also possible to deliver rDNA-derived nucleases to cells as RNA or protein, and so foreign DNA would not need to be introduced into plants to achieve the desired mutagenic outcome. In such cases, the rDNA molecule itself never encounters a plant cell. More direction is required from regulatory agencies to stipulate the way in which rDNA can be used in the process of generating crops before the regulated status is triggered. TagMo-derived crops that introduce alien transgenes or knock out native genes are similar to traditional GM crops or conventionally mutagenized plants, respectively, but TagMo crops that alter the DNA sequence of the target gene (Fig 1D) are more difficult to classify. For example, a GM plant could have a single nucleotide change that distinguishes it from its parent and that confers a new trait such as herbicide resistance. If such a subtle genetic alteration were attained by traditional mutagenesis or by screening for natural variation, the resulting plants would not be regulated. As discussed above, if rDNA techniques are used to create the single nucleotide TagMo, one could argue that it should be regulated. Regulation would then focus on the process rather than the product. If single nucleotide changes were exempt, would there be a threshold in the number of bases that can be modified before concerns are raised or regulatory scrutiny is triggered? Or would there be a difference in regulation if the gene replacement involves a sexually compatible or an incompatible species? Most of this discussion has focused on the use of engineered nucleases such as meganucleases or zinc-finger nucleases to create TagMos. Oligonucleotide-mediated mutagenesis (OMM), however, is also used to modify plant genes (Breyer et al, 2009). OMM uses chemically synthesized oligonucleotides that are homologous to the target gene, except for the nucleotides to be changed. Breyer et al (2009) argue that OMM “should not be considered as using recombinant nucleic acid molecules” and that “OMM should be considered as a form of mutagenesis, a technique which is excluded from the scope of the EU regulation.” However, they admit that the resulting crops could be considered as GM organisms, according to EU regulatory definitions for biotechnology. They report that in the USA, OMM plants have been declared non-GM by the USDA, but it is unclear whether the non-GM distinction in the USA has regulatory implications. OMM is already being used to develop crops with herbicide tolerance, and so regulatory guidelines need to be clarified before market release. In turning to address how TagMo-related oversight should proceed, two questions are central: how are decisions made and who is involved in making them? The analysis above illustrates that many fundamental decisions need to be made concerning the way in which TagMo-derived products will be regulated and, more broadly, what constitutes a GM organism for regulatory purposes. These decisions are inherently values-based in that views on how to regulate TagMo products differ on the basis of understanding of and attitudes towards agriculture, risk, nature and technology. Neglecting the values-based assumptions underlying these decisions can lead to poor decision-making, through misunderstanding the issues at hand, and public and stakeholder backlash resulting from disagreements over values. Bozeman & Sarewitz (2005) consider this problem in a framework of ‘market failures’ and ‘public failures’. GM crops have exhibited both. Market failures are exemplified by the loss of trade with the EU owing to different regulatory standards and levels of caution (PIFB, 2006). Furthermore, there has been a decline in the number of GM crops approved for interstate movement in the USA since 2001. Public failures result from incongruence between actions by decision-makers and the values of the public. Public failures are exemplified by the anti-GM sentiment in the labelling of organic foods in the USA and court challenges to the biosafety review of GM crops by the USDA's Animal and Plant Health Inspection Service (McHughen & Smyth, 2008). These lawsuits have delayed approval of genetically engineered alfalfa and sugar beet, thus blurring the distinction between public and market failures. Public failures will probably ensue if TagMo crops slip into the market under the radar without adequate oversight. The possibility of public failures with TagMo crops highlights the benefits of an anticipatory governance-based approach, and will help to ensure that the technology meets societal needs Anticipatory governance is a framework with principles that are well suited to guiding TagMo-related oversight and to helping to avoid public failures. It challenges an understanding of technology development that downplays the importance of societal factors—such as implications for stakeholders and the environment—and argues that societal factors should inform technology development and governance from the start (Macnaghten et al, 2005). Anticipatory governance uses three principles: foresight, integration of natural and social science research, and upstream public engagement (Karinen & Guston, 2010). The first two principles emphasize proactive engagement using interdisciplinary knowledge. Governance processes that use these principles include real-time technology assessment (Guston & Sarewitz, 2002) and upstream oversight assessment (Kuzma et al, 2008b). The third principle, upstream public engagement, involves stakeholders and the public in directing values-based assumptions within technology development and oversight (Wilsdon & Wills, 2004). Justifications for upstream public engagement are substantive (stakeholders and the public can provide information that improves decisions), instrumental (including stakeholders and the public in the decision-making process leads to more trusted decisions) and normative (citizens have a right to influence decisions about issues that affect them). TagMo crop developers seem to be arguing for a ‘process-based’ exclusion of TagMo crops from regulatory oversight, without public knowledge of their development or ongoing regulatory communication. We propose that the discussion about how to regulate TagMo crops should be open, use public engagement and respect several criteria of oversight (Kuzma et al, 2008a). These criteria should include not only biosafety, but also broader impacts on human and ecological health and well-being, distribution of health impacts, transparency, treatment of intellectual property and confidential business information, economic costs and benefits, as well as public confidence and values. We also propose that the CFRB should be a starting point for TagMo oversight. The various categories of TagMo require an approach that can discern and address the risks associated with each application. The CFRB allows for such flexibility. At the same time, the CFRB should improve public engagement and transparency, post-market monitoring and some aspects of technical risk assessment. As we have argued, TagMo is on the verge of being broadly implemented to create crop varieties with new traits, and this raises many oversight questions. First, the way in which TagMo technologies will be classified and handled within the US regulatory system has yet to be determined. As these decisions are complex, values-based and have far-reaching implications, they should be made in a transparent way that draws on insights from the natural and social sciences, and involves stakeholders and the public. Second, as products derived from TagMo technologies will soon reach the marketplace, it is important to begin predicting and addressing potential regulatory challenges, to ensure that oversight systems are in place. The possibility of public failures with TagMo crops highlights the benefits of an anticipatory governance-based approach, and will help to ensure that the technology meets societal needs. So far, the EU has emphasized governance approaches and stakeholder involvement in the regulation of new technologies more than the USA. However, if the USA can agree on a regulatory system for TagMo crops that is the result of open and transparent discussions with the public and stakeholders, it could take the lead and act as a model for similar regulation in the EU and globally. Before this can happen, a shift in US approaches to regulatory policy would be needed. Acknowledgements This work was supported by National Science Foundation grant 0923827. The authors thank Professor D. Voytas for helpful technical input in the preparation of the manuscript and M. Christian for technical conversations during the preparation of this manuscript. Conflict of Interest The authors declare that they have no conflict of interest. Biographies Jennifer Kuzma is at the Humphrey School of Public Affairs and Center for Science, University of Minnesota, Minneapolis, USA.. E-mail: [email protected] Adam Kokotovich is a PhD student in the Natural Resources Science & Management Program and a research assistant at the Center for Science, Technology & Public Policy, University of Minnesota, St Paul, USA. E-mail: [email protected] References Bozeman B, Sarewitz D (2005) Public values and public failures in US science policy. Sci Public Pol 32: 119–136CrossrefGoogle Scholar Breyer D, Herman P, Brandenburger A, Gheysen G, Remaut E, Soumillion P, Van Doorsselaere J, Custers R, Pauwels K, Sneyers M (2009) Commentary: genetic modification through oligonucleotide-mediated mutagenesis. A GMO regulatory challenge? Environ Biosafety Res 8: 57–64CrossrefPubMedGoogle Scholar EPA (2001) Regulations under the Federal Fungicide, Insecticide, and Rodenticide Act for plant-incorporated protectants. Federal Register 66: 37772Google Scholar Gao H, Smith J, Yang M, Jones S, Djukanovic V, Nicholson MG, West A, Bidney D, Falco SC, Jantz D (2010) Heritable targeted mutagenesis in maize using a designed endonuclease. Plant J 61: 176–187Wiley Online LibraryCASPubMedWeb of Science®Google Scholar Guston DH, Sarewitz D (2002) Real-time technology assessment. Technol Soc 23: 93–109CrossrefGoogle Scholar Karinen R, Guston DH (2010) Toward anticipatory governance: the experience with nanotechnology. In Governing Future Technologies: Nanotechnology and the Rise of an Assessment Regime, Kaiser M, Kurath M, Maasen S, Rehmann-Sutter C (eds), pp 217–232. Dordrecht, The Netherlands: SpringerWeb of Science®Google Scholar Kuzma J, Meghani Z (2009) The public option. EMBO Rep 10: 1288–1293Wiley Online LibraryCASPubMedWeb of Science®Google Scholar Kuzma J, Najmaie P, Larson J (2009) Evaluating oversight systems for emerging technologies: a case study of genetically engineered organisms. J Law Med Ethics 37: 546–586Wiley Online LibraryCASWeb of Science®Google Scholar Kuzma J, Paradise J, Ramachandran G, Kim J, Kokotovich A, Wolf SM (2008a) An integrated approach to oversight assessment for emerging technologies. Risk Analysis 28: 1197–1219Wiley Online LibraryCASPubMedWeb of Science®Google Scholar Kuzma J, Romanchek J, Kokotovich A (2008b) Upstream oversight assessment for agrifood nanotechnology: a case studies approach. Risk Analysis 28: 1081–1098Wiley Online LibraryCASPubMedWeb of Science®Google Scholar Macnaghten P, Kearnes MB, Wynne B (2005) Nanotechnology, governance, and public deliberation: what role for the social sciences? Sci Commun 27: 268CrossrefWeb of Science®Google Scholar Magana-Gomez J, de la Barca AMC (2009) Risk assessment of genetically modified crops for nutrition and health. Nutrition Rev 67: 1–16Wiley Online LibraryPubMedWeb of Science®Google Scholar McHughen A, Smyth S (2008) US regulatory system for genetically modified [genetically modified organism (GMO), rDNA or transgenic] crop cultivars. Plant Biotechnol 6: 2–12Wiley Online LibraryPubMedWeb of Science®Google Scholar National Research Council (1989) Field Testing Genetically Modified Organisms: Framework for Decisions. Washington, DC, USA: National Academies PressGoogle Scholar National Research Council (2000) Genetically Modified Pest-Protected Plants: Science and Regulation. Washington, DC, USA: National Academies PressGoogle Scholar National Research Council (2002) Environmental Effects of Transgenic Plants. Washington, DC, USA: National Academies PressGoogle Scholar National Research Council (2004) Safety of Genetically Engineered Foods. Wasington, DC, USA: National Academies PressGoogle Scholar Office of Science and Technology Policy (1986) Coordinated framework for the regulation of biotechnology. Federal Register 51: 23302PubMedGoogle Scholar PIFB (2006) Agricultural Biotechnology Information Disclosure: Accommodating Conflicting Interests Within Public Access Norms. Philadelphia, PA, USA: Pew Initiative on Food and BiotechnologyGoogle Scholar Pollack A (2011) US says farmers may grow engineered sugar beets. The New York Times 4 FebGoogle Scholar Russell AW, Sparrow R (2008) The case for regulating intragenic GMOs. J Agr Environ Ethics 21: 153–181CrossrefWeb of Science®Google Scholar Schouten HJ, Krens FA, Jacobsen E (2006) Cisgenic plants are similar to traditionally bred plants. EMBO Rep 7: 750–753Wiley Online LibraryCASPubMedWeb of Science®Google Scholar Shukla VK, Doyon Y, Miller JC, DeKelver RC, Moehle EA, Worden SE, Mitchell JC, Arnold NL, Gopalan S, Meng X (2009) Precise genome modification in the crop species Zea mays using zinc-finger nucleases. Nature 459: 437–441CrossrefCASPubMedWeb of Science®Google Scholar Thompson PB (2007) Food Biotechnology in Ethical Perspective. Dordrecht, The Netherlands: SpringerGoogle Scholar Townsend JA, Wright DA, Winfrey RJ, Fu F, Maeder ML, Joung JK, Voytas DF (2009) High-frequency modification of plant genes using engineered zinc-finger nucleases. Nature 459: 442–445CrossrefCASPubMedWeb of Science®Google Scholar USDA (1997) Introduction of Organisms and Products Altered or Produced Through Genetic Engineering which are Plant Pests or Which There is a Reason to Believe are Plant Pests. 7 CFR 340. http://ecfr.gpoaccess.govGoogle Scholar USDA (2010) Adoption of Genetically Engineered Crops in the US. http://www.ers.usda.gov/Data/BiotechCropsGoogle Scholar Voosen P (2011) USDA's alfalfa decision postpones reckoning on biotech crops. The New York Times 28 JanGoogle Scholar Weinthal D, Tovkach A, Zeevi V, Tzfira T (2010) Genome editing in plant cells by zinc finger nucleases. Trends Plant Sci 15: 308–321CrossrefCASPubMedWeb of Science®Google Scholar Wilsdon J, Wills R (2004) See-through science: why public engagement needs to move upstream. London, UK: Demos. www.demos.co.ukWeb of Science®Google Scholar Previous ArticleNext Article Volume 12Issue 91 September 2011In this issue FiguresReferencesRelatedDetailsLoading ...}, number={9}, journal={EMBO reports}, publisher={Wiley}, author={Kuzma, Jennifer and Kokotovich, Adam}, year={2011}, month={Aug}, pages={883–888} }
 @article{kuzma_paradise_ramachandran_kim_kokotovich_wolf_2008, title={An Integrated Approach to Oversight Assessment for Emerging Technologies}, volume={28}, ISSN={0272-4332 1539-6924}, url={http://dx.doi.org/10.1111/j.1539-6924.2008.01086.x}, DOI={10.1111/j.1539-6924.2008.01086.x}, abstractNote={Analysis of oversight systems is often conducted from a single disciplinary perspective and by using a limited set of criteria for evaluation. In this article, we develop an approach that blends risk analysis, social science, public administration, legal, public policy, and ethical perspectives to develop a broad set of criteria for assessing oversight systems. Multiple methods, including historical analysis, expert elicitation, and behavioral consensus, were employed to develop multidisciplinary criteria for evaluating oversight of emerging technologies. Sixty‐six initial criteria were identified from extensive literature reviews and input from our Working Group. Criteria were placed in four categories reflecting the development, attributes, evolution, and outcomes of oversight systems. Expert elicitation, consensus methods, and multidisciplinary review of the literature were used to refine a condensed, operative set of criteria. Twenty‐eight criteria resulted spanning four categories: seven development criteria, 15 attribute criteria, five outcome criteria, and one evolution criterion. These criteria illuminate how oversight systems develop, operate, change, and affect society. We term our approach “integrated oversight assessment” and propose its use as a tool for analyzing relationships among features, outcomes, and tradeoffs of oversight systems. Comparisons among historical case studies of oversight using a consistent set of criteria should result in defensible and evidence‐supported lessons to guide the development of oversight systems for emerging technologies, such as nanotechnology.}, number={5}, journal={Risk Analysis}, publisher={Wiley}, author={Kuzma, Jennifer and Paradise, Jordan and Ramachandran, Gurumurthy and Kim, Jee-Ae and Kokotovich, Adam and Wolf, Susan M.}, year={2008}, month={Oct}, pages={1197–1220} }
 @article{kuzma_romanchek_kokotovich_2008, title={Upstream Oversight Assessment for Agrifood Nanotechnology: A Case Studies Approach}, volume={28}, ISSN={["1539-6924"]}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-48349093922&partnerID=MN8TOARS}, DOI={10.1111/j.1539-6924.2008.01071.x}, abstractNote={Although nanotechnology is broadly receiving attention in public and academic circles, oversight issues associated with applications for agriculture and food remain largely unexplored. Agrifood nanotechnology is at a critical stage in which informed analysis can help shape funding priorities, risk assessment, and oversight activities. This analysis is designed to help society and policymakers anticipate and prepare for challenges posed by complicated, convergent applications of agrifood nanotechnology. The goal is to identify data, risk assessment, regulatory policy, and engagement needs for overseeing these products so they can be addressed prior to market entry. Our approach, termed upstream oversight assessment (UOA), has potential as a key element of anticipatory governance. It relies on distinct case studies of proposed applications of agrifood nanotechnology to highlight areas that need study and attention. As a tool for preparation, UOA anticipates the types and features of emerging applications; their endpoints of use in society; the extent to which users, workers, ecosystems, or consumers will be exposed; the nature of the material and its safety; whether and where the technologies might fit into current regulatory system(s); the strengths and weaknesses of the system(s) in light of these novel applications; and the possible social concerns related to oversight for them.}, number={4}, journal={Risk Analysis}, author={Kuzma, J. and Romanchek, J. and Kokotovich, A.}, year={2008}, pages={1081–1098} }