@article{gao_si_wen_li_huang_2021, title={Reinforcement Learning Control of Robotic Knee With Human-in-the-Loop by Flexible Policy Iteration}, volume={5}, ISSN={["2162-2388"]}, url={http://dx.doi.org/10.1109/tnnls.2021.3071727}, DOI={10.1109/TNNLS.2021.3071727}, abstractNote={We are motivated by the real challenges presented in a human–robot system to develop new designs that are efficient at data level and with performance guarantees, such as stability and optimality at system level. Existing approximate/adaptive dynamic programming (ADP) results that consider system performance theoretically are not readily providing practically useful learning control algorithms for this problem, and reinforcement learning (RL) algorithms that address the issue of data efficiency usually do not have performance guarantees for the controlled system. This study fills these important voids by introducing innovative features to the policy iteration algorithm. We introduce flexible policy iteration (FPI), which can flexibly and organically integrate experience replay and supplemental values from prior experience into the RL controller. We show system-level performances, including convergence of the approximate value function, (sub)optimality of the solution, and stability of the system. We demonstrate the effectiveness of the FPI via realistic simulations of the human–robot system. It is noted that the problem we face in this study may be difficult to address by design methods based on classical control theory as it is nearly impossible to obtain a customized mathematical model of a human–robot system either online or offline. The results we have obtained also indicate the great potential of RL control to solving realistic and challenging problems with high-dimensional control inputs.}, number={10}, journal={IEEE TRANSACTIONS ON NEURAL NETWORKS AND LEARNING SYSTEMS}, publisher={Institute of Electrical and Electronics Engineers (IEEE)}, author={Gao, Xiang and Si, Jennie and Wen, Yue and Li, Minhan and Huang, He}, year={2021}, month={May} }
 @article{li_wen_gao_si_huang_2021, title={Toward Expedited Impedance Tuning of a Robotic Prosthesis for Personalized Gait Assistance by Reinforcement Learning Control}, volume={5}, ISSN={["1941-0468"]}, url={http://dx.doi.org/10.1109/tro.2021.3078317}, DOI={10.1109/TRO.2021.3078317}, abstractNote={Personalizing medical devices such as lower limb wearable robots is challenging. While the initial feasibility of automating the process of knee prosthesis control parameter tuning has been demonstrated in a principled way, the next critical issue is to improve tuning efficiency and speed it up for the human user, in clinic settings, while maintaining human safety. We, therefore, propose a policy iteration with constraint embedded (PICE) method as an innovative solution to the problem under the framework of reinforcement learning. Central to PICE is the use of a projected Bellman equation with a constraint of assuring positive semidefiniteness of performance values during policy evaluation. Additionally, we developed both online and offline PICE implementations that provide additional flexibility for the designer to fully utilize measurement data, either from on-policy or off-policy, to further improve PICE tuning efficiency. Our human subject testing showed that the PICE provided effective policies with significantly reduced tuning time. For the first time, we also experimentally evaluated and demonstrated the robustness of the deployed policies by applying them to different tasks and users. Putting it together, our new way of problem solving has been effective as PICE has demonstrated its potential toward truly automating the process of control parameter tuning for robotic knee prosthesis users.}, number={1}, journal={IEEE TRANSACTIONS ON ROBOTICS}, publisher={Institute of Electrical and Electronics Engineers (IEEE)}, author={Li, Minhan and Wen, Yue and Gao, Xiang and Si, Jennie and Huang, He}, year={2021}, month={May} }
 @article{wen_li_si_huang_2020, title={Wearer-Prosthesis Interaction for Symmetrical Gait: A Study Enabled by Reinforcement Learning Prosthesis Control}, volume={28}, ISSN={["1558-0210"]}, url={http://www.scopus.com/inward/record.url?eid=2-s2.0-85083163810&partnerID=MN8TOARS}, DOI={10.1109/TNSRE.2020.2979033}, abstractNote={With advances in robotic prostheses, rese-archers attempt to improve amputee’s gait performance (e.g., gait symmetry) beyond restoring normative knee kinematics/kinetics. Yet, little is known about how the prosthesis mechanics/control influence wearer-prosthesis’ gait performance, such as gait symmetry, stability, etc. This study aimed to investigate the influence of robotic transfemoral prosthesis mechanics on human wearers’ gait symmetry. The investigation was enabled by our previously designed reinforcement learning (RL) supplementary control, which simultaneously tuned 12 control parameters that determined the prosthesis mechanics throughout a gait cycle. The RL control design facilitated safe explorations of prosthesis mechanics with the human in the loop. Subjects were recruited and walked with a robotic transfemoral prosthesis on a treadmill while the RL controller tuned the control parameters. Stance time symmetry, step length symmetry, and bilateral anteroposterior (AP) impulses were measured. The data analysis showed that changes in robotic knee mechanics led to movement variations in both lower limbs and therefore gait temporal-spatial symmetry measures. Consistent across all the subjects, inter-limb AP impulse measurements explained gait symmetry: the stance time symmetry was significantly correlated with the net inter-limb AP impulse, and the step length symmetry was significantly correlated with braking and propulsive impulse symmetry. The results suggest that it is possible to personalize transfemoral prosthesis control for improved temporal-spatial gait symmetry. However, adjusting prosthesis mechanics alone was insufficient to maximize the gait symmetry. Rather, achieving gait symmetry may require coordination between the wearer’s motor control of the intact limb and adaptive control of the prosthetic joints. The results also indicated that the RL-based prosthesis tuning system was a potential tool for studying wearer-prosthesis interactions.}, number={4}, journal={IEEE TRANSACTIONS ON NEURAL SYSTEMS AND REHABILITATION ENGINEERING}, author={Wen, Yue and Li, Minhan and Si, Jennie and Huang, He}, year={2020}, month={Apr}, pages={904–913} }
 @article{wen_si_gao_huang_huang_2017, title={A new powered lower limb prosthesis control framework based on adaptive dynamic programming}, volume={28}, number={9}, journal={IEEE Transactions on Neural Networks and Learning Systems}, author={Wen, Y. and Si, J. and Gao, X. and Huang, S. and Huang, H.}, year={2017}, pages={2215–2220} }
 @article{brandt_wen_liu_stallings_huang_2017, title={Interactions between transfemoral amputees and a powered knee prosthesis during load carriage}, volume={7}, journal={Scientific Reports}, author={Brandt, A. and Wen, Y. and Liu, M. and Stallings, J. and Huang, H. H.}, year={2017} }