@article{alili_nalam_li_liu_feng_si_huang_2023, title={A Novel Framework to Facilitate User Preferred Tuning for a Robotic Knee Prosthesis}, volume={31}, ISSN={["1558-0210"]}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-85147231065&partnerID=MN8TOARS}, DOI={10.1109/TNSRE.2023.3236217}, abstractNote={The tuning of robotic prosthesis control is essential to provide personalized assistance to individual prosthesis users. Emerging automatic tuning algorithms have shown promise to ease the device personalization procedure. However, very few automatic tuning algorithms consider the user preference as the tuning goal, which may limit the adoptability of the robotic prosthesis. In this study, we propose and evaluate a novel prosthesis control tuning framework for a robotic knee prosthesis, which could enable user preferred robot behavior in the device tuning process. The framework consists of 1) a User-Controlled Interface that allows the user to select their preferred knee kinematics in gait and 2) a reinforcement learning-based algorithm for tuning high-dimension prosthesis control parameters to meet the desired knee kinematics. We evaluated the performance of the framework along with usability of the developed user interface. In addition, we used the developed framework to investigate whether amputee users can exhibit a preference between different profiles during walking and whether they can differentiate between their preferred profile and other profiles when blinded. The results showed effectiveness of our developed framework in tuning 12 robotic knee prosthesis control parameters while meeting the user-selected knee kinematics. A blinded comparative study showed that users can accurately and consistently identify their preferred prosthetic control knee profile. Further, we preliminarily examined gait biomechanics of the prosthesis users when walking with different prosthesis control and did not find clear difference between walking with preferred prosthesis control and when walking with normative gait control parameters. This study may inform future translation of this novel prosthesis tuning framework for home or clinical use.}, journal={IEEE TRANSACTIONS ON NEURAL SYSTEMS AND REHABILITATION ENGINEERING}, author={Alili, Abbas and Nalam, Varun and Li, Minhan and Liu, Ming and Feng, Jing and Si, Jennie and Huang, He}, year={2023}, pages={895–903} }
 @article{li_liu_si_stallrich_huang_2023, title={Hierarchical Optimization for Control of Robotic Knee Prostheses Toward Improved Symmetry of Propulsive Impulse}, volume={70}, ISSN={["1558-2531"]}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-85142785855&partnerID=MN8TOARS}, DOI={10.1109/TBME.2022.3224026}, abstractNote={Automatically personalizing complex control of robotic prostheses to improve gait performance, such as gait symmetry, is challenging. Recently, human-in-the-loop (HIL) optimization and reinforcement learning (RL) have shown promise in achieving optimized control of wearable robots for each individual user. However, HIL optimization methods lack scalability for high-dimensional space, while RL has mostly focused on optimizing robot kinematic performance. Thus, we propose a novel hierarchical framework to personalize robotic knee prosthesis control and improve overall gait performance. Specifically, in this study the framework was implemented to simultaneously design target knee kinematics and tune 12 impedance control parameters for improved symmetry of propulsive impulse in walking. In our proposed framework, HIL optimization is used to identify an optimal target knee kinematics with respect to symmetry improvement, while RL is leveraged to yield an optimal policy for tuning impedance parameters in high-dimensional space to match the kinematics target. The proposed framework was validated on human subjects, walking with robotic knee prosthesis. The results showed that our design successfully shaped the target knee kinematics as well as configured 12 impedance control parameters to improve propulsive impulse symmetry of the human users. The knee kinematics that yielded best propulsion symmetry did not preserve the normative knee kinematics profile observed in non-disabled individuals, suggesting that restoration of normative joint biomechanics in walking does not necessarily optimize the gait performance of human-prosthesis systems. This new framework for prosthesis control personalization may be extended to other wearable devices or different gait performance optimization goals in the future.}, number={5}, journal={IEEE TRANSACTIONS ON BIOMEDICAL ENGINEERING}, author={Li, Minhan and Liu, Wentao and Si, Jennie and Stallrich, Jonathan W. and Huang, He}, year={2023}, month={May}, pages={1634–1642} }
 @article{liu_zhong_wu_fylstra_si_huang_2022, title={Inferring Human-Robot Performance Objectives During Locomotion Using Inverse Reinforcement Learning and Inverse Optimal Control}, volume={7}, ISSN={["2377-3766"]}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-85123342480&partnerID=MN8TOARS}, DOI={10.1109/LRA.2022.3143579}, abstractNote={Quantitatively characterizing a locomotion performance objective for a human-robot system is an important consideration in the assistive wearable robot design towards human-robot symbiosis. This problem, however, has only been addressed sparsely in the literature. In this study, we propose a new inverse approach from observed human-robot walking behavior to infer a human-robot collective performance objective represented in a quadratic form. By an innovative design of human experiments and simulation study, respectively, we validated the effectiveness of two solution approaches to solving the inverse problem using inverse reinforcement learning (IRL) and inverse optimal control (IOC). The IRL-based experiments of human walking with robotic transfemoral prosthesis validated the realistic applicability of the proposed inverse approach, while the IOC-based analysis provided important human-robot system properties such as stability and robustness that are difficult to obtain from human experiments. This study introduces a new tool to the field of wearable lower limb robots. It is expected to be expandable to quantify joint human-robot locomotion performance objectives for personalizing wearable robot control in the future.}, number={2}, journal={IEEE ROBOTICS AND AUTOMATION LETTERS}, author={Liu, Wentao and Zhong, Junmin and Wu, Ruofan and Fylstra, Bretta L. and Si, Jennie and Huang, He}, year={2022}, month={Apr}, pages={2549–2556} }
 @article{wu_li_yao_liu_si_huang_2022, title={Reinforcement Learning Impedance Control of a Robotic Prosthesis to Coordinate With Human Intact Knee Motion}, volume={7}, ISSN={["2377-3766"]}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-85131742245&partnerID=MN8TOARS}, DOI={10.1109/LRA.2022.3179420}, abstractNote={This study aims to demonstrate reinforcement learning tracking control for automatically configuring the impedance parameters of a robotic knee prosthesis. While our previous studies involving human subjects have focused on tuning the impedance control parameters to meet a fixed, subjectively prescribed target motion profile to enable continuous walking with human-in-the-loop, in this paper we develop a new tracking control solution for a robotic knee to mimic the motion of the intact knee. As such, we replaced the prescribed target knee motion by an automatically generated profile based on the intact knee. As the profile of the intact knee varies over time due to human adaptation, we are presented with a challenging tracking control problem in the context of classical control theory. By formulating the “echo control” of the robotic knee as a reinforcement learning problem, we provide a promising new tool for real-time tracking control design without explicitly representing the underlying dynamics using a mathematical model, which can be difficult to obtain for a human-robot system. Additionally, our results may inspire future studies and new robotic prosthesis impedance control designs that can potentially coordinate between the intact and the robotic limbs toward daily use of the robotic device.}, number={3}, journal={IEEE ROBOTICS AND AUTOMATION LETTERS}, author={Wu, Ruofan and Li, Minhan and Yao, Zhikai and Liu, Wentao and Si, Jennie and Huang, He}, year={2022}, month={Jul}, pages={7014–7020} }
 @article{wu_yao_si_huang_2022, title={Robotic Knee Tracking Control to Mimic the Intact Human Knee Profile Based on Actor-Critic Reinforcement Learning}, volume={9}, ISSN={["2329-9274"]}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-85118313741&partnerID=MN8TOARS}, DOI={10.1109/JAS.2021.1004272}, abstractNote={We address a state-of-the-art reinforcement learning (RL) control approach to automatically configure robotic prosthesis impedance parameters to enable end-to-end, continuous locomotion intended for transfemoral amputee subjects. Specifically, our actor-critic based RL provides tracking control of a robotic knee prosthesis to mimic the intact knee profile, This is a significant advance from our previous RL based automatic tuning of prosthesis control parameters which have centered on regulation control with a designer prescribed robotic knee profile as the target. In addition to presenting the tracking control algorithm based on direct heuristic dynamic programming (dHDP), we provide a control performance guarantee including the case of constrained inputs. We show that our proposed tracking control possesses several important properties, such as weight convergence of the learning networks, Bellman (sub) optimality of the cost-to-go value function and control input, and practical stability of the human-robot system. We further provide a systematic simulation of the proposed tracking control using a realistic human-robot system simulator, the OpenSim, to emulate how the dHDP enables level ground walking, walking on different terrains and at different paces. These results show that our proposed dHDP based tracking control is not only theoretically suitable, but also practically useful.}, number={1}, journal={IEEE-CAA JOURNAL OF AUTOMATICA SINICA}, author={Wu, Ruofan and Yao, Zhikai and Si, Jennie and Huang, He Helen}, year={2022}, month={Jan}, pages={19–30} }
 @article{gao_si_wen_li_huang_2021, title={Reinforcement Learning Control of Robotic Knee With Human-in-the-Loop by Flexible Policy Iteration}, volume={5}, ISSN={["2162-2388"]}, url={http://dx.doi.org/10.1109/tnnls.2021.3071727}, DOI={10.1109/TNNLS.2021.3071727}, abstractNote={We are motivated by the real challenges presented in a human–robot system to develop new designs that are efficient at data level and with performance guarantees, such as stability and optimality at system level. Existing approximate/adaptive dynamic programming (ADP) results that consider system performance theoretically are not readily providing practically useful learning control algorithms for this problem, and reinforcement learning (RL) algorithms that address the issue of data efficiency usually do not have performance guarantees for the controlled system. This study fills these important voids by introducing innovative features to the policy iteration algorithm. We introduce flexible policy iteration (FPI), which can flexibly and organically integrate experience replay and supplemental values from prior experience into the RL controller. We show system-level performances, including convergence of the approximate value function, (sub)optimality of the solution, and stability of the system. We demonstrate the effectiveness of the FPI via realistic simulations of the human–robot system. It is noted that the problem we face in this study may be difficult to address by design methods based on classical control theory as it is nearly impossible to obtain a customized mathematical model of a human–robot system either online or offline. The results we have obtained also indicate the great potential of RL control to solving realistic and challenging problems with high-dimensional control inputs.}, number={10}, journal={IEEE TRANSACTIONS ON NEURAL NETWORKS AND LEARNING SYSTEMS}, publisher={Institute of Electrical and Electronics Engineers (IEEE)}, author={Gao, Xiang and Si, Jennie and Wen, Yue and Li, Minhan and Huang, He}, year={2021}, month={May} }
 @article{huang_si_brandt_li_2021, title={Taking both sides: seeking symbiosis between intelligent prostheses and human motor control during locomotion}, volume={20}, ISSN={["2468-4511"]}, url={http://dx.doi.org/10.1016/j.cobme.2021.100314}, DOI={10.1016/j.cobme.2021.100314}, abstractNote={Robotic lower-limb prostheses aim to replicate the power-generating capability of biological joints during locomotion to empower individuals with lower-limb loss. However, recent clinical trials have not demonstrated clear advantages of these devices over traditional passive devices. We believe this is partly because the current designs of robotic prothesis controllers and clinical methods for fitting and training individuals to use them do not ensure good coordination between the prosthesis and user. Accordingly, we advocate for new holistic approaches in which human motor control and intelligent prosthesis control function as one system (defined as human-prosthesis symbiosis). We hope engineers and clinicians will work closely to achieve this symbiosis, thereby improving the functionality and acceptance of robotic prostheses and users' quality of life.}, journal={CURRENT OPINION IN BIOMEDICAL ENGINEERING}, publisher={Elsevier BV}, author={Huang, He and Si, Jennie and Brandt, Andrea and Li, Minhan}, year={2021}, month={Dec} }
 @article{li_wen_gao_si_huang_2021, title={Toward Expedited Impedance Tuning of a Robotic Prosthesis for Personalized Gait Assistance by Reinforcement Learning Control}, volume={5}, ISSN={["1941-0468"]}, url={http://dx.doi.org/10.1109/tro.2021.3078317}, DOI={10.1109/TRO.2021.3078317}, abstractNote={Personalizing medical devices such as lower limb wearable robots is challenging. While the initial feasibility of automating the process of knee prosthesis control parameter tuning has been demonstrated in a principled way, the next critical issue is to improve tuning efficiency and speed it up for the human user, in clinic settings, while maintaining human safety. We, therefore, propose a policy iteration with constraint embedded (PICE) method as an innovative solution to the problem under the framework of reinforcement learning. Central to PICE is the use of a projected Bellman equation with a constraint of assuring positive semidefiniteness of performance values during policy evaluation. Additionally, we developed both online and offline PICE implementations that provide additional flexibility for the designer to fully utilize measurement data, either from on-policy or off-policy, to further improve PICE tuning efficiency. Our human subject testing showed that the PICE provided effective policies with significantly reduced tuning time. For the first time, we also experimentally evaluated and demonstrated the robustness of the deployed policies by applying them to different tasks and users. Putting it together, our new way of problem solving has been effective as PICE has demonstrated its potential toward truly automating the process of control parameter tuning for robotic knee prosthesis users.}, number={1}, journal={IEEE TRANSACTIONS ON ROBOTICS}, publisher={Institute of Electrical and Electronics Engineers (IEEE)}, author={Li, Minhan and Wen, Yue and Gao, Xiang and Si, Jennie and Huang, He}, year={2021}, month={May} }
 @article{alili_nalam_li_liu_si_huang_2021, title={User Controlled Interface for Tuning Robotic Knee Prosthesis}, ISSN={["2153-0858"]}, url={http://dx.doi.org/10.1109/iros51168.2021.9636264}, DOI={10.1109/IROS51168.2021.9636264}, abstractNote={The tuning process for a robotic prosthesis is a challenging and time-consuming task both for users and clinicians. An automatic tuning approach using reinforcement learning (RL) has been developed for a knee prosthesis to address the challenges of manual tuning methods. The algorithm tunes the optimal control parameters based on the provided knee joint profile that the prosthesis is expected to replicate during gait safely. This paper presents an intuitive interface designed for the prosthesis users and clinicians to choose the preferred knee joint profile during gait and use the autotuner to replicate in the prosthesis. The interface-based approach is validated by observing the ability of the tuning algorithm to successfully converge to various alternate knee profiles by testing on two able-bodied subjects walking with a robotic knee prosthesis. The algorithm was found to converge successfully in an average duration of 1.15 min for the first subject and 2.31 min for the second subject. Further, the subjects displayed different preferences for optimal profiles reinforcing the need to tune alternate profiles. The implications of the results in the tuning of robotic prosthetic devices are discussed.}, journal={2021 IEEE/RSJ INTERNATIONAL CONFERENCE ON INTELLIGENT ROBOTS AND SYSTEMS (IROS)}, publisher={IEEE}, author={Alili, Abbas and Nalam, Varun and Li, Minhan and Liu, Ming and Si, Jennie and Huang, He}, year={2021}, pages={6190–6195} }
 @article{wen_li_si_huang_2020, title={Wearer-Prosthesis Interaction for Symmetrical Gait: A Study Enabled by Reinforcement Learning Prosthesis Control}, volume={28}, ISSN={["1558-0210"]}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-85083163810&partnerID=MN8TOARS}, DOI={10.1109/TNSRE.2020.2979033}, abstractNote={With advances in robotic prostheses, rese-archers attempt to improve amputee’s gait performance (e.g., gait symmetry) beyond restoring normative knee kinematics/kinetics. Yet, little is known about how the prosthesis mechanics/control influence wearer-prosthesis’ gait performance, such as gait symmetry, stability, etc. This study aimed to investigate the influence of robotic transfemoral prosthesis mechanics on human wearers’ gait symmetry. The investigation was enabled by our previously designed reinforcement learning (RL) supplementary control, which simultaneously tuned 12 control parameters that determined the prosthesis mechanics throughout a gait cycle. The RL control design facilitated safe explorations of prosthesis mechanics with the human in the loop. Subjects were recruited and walked with a robotic transfemoral prosthesis on a treadmill while the RL controller tuned the control parameters. Stance time symmetry, step length symmetry, and bilateral anteroposterior (AP) impulses were measured. The data analysis showed that changes in robotic knee mechanics led to movement variations in both lower limbs and therefore gait temporal-spatial symmetry measures. Consistent across all the subjects, inter-limb AP impulse measurements explained gait symmetry: the stance time symmetry was significantly correlated with the net inter-limb AP impulse, and the step length symmetry was significantly correlated with braking and propulsive impulse symmetry. The results suggest that it is possible to personalize transfemoral prosthesis control for improved temporal-spatial gait symmetry. However, adjusting prosthesis mechanics alone was insufficient to maximize the gait symmetry. Rather, achieving gait symmetry may require coordination between the wearer’s motor control of the intact limb and adaptive control of the prosthetic joints. The results also indicated that the RL-based prosthesis tuning system was a potential tool for studying wearer-prosthesis interactions.}, number={4}, journal={IEEE TRANSACTIONS ON NEURAL SYSTEMS AND REHABILITATION ENGINEERING}, author={Wen, Yue and Li, Minhan and Si, Jennie and Huang, He}, year={2020}, month={Apr}, pages={904–913} }