@article{brown_harlim_2013, title={Assimilating irregularly spaced sparsely observed turbulent signals with hierarchical Bayesian reduced stochastic filters}, volume={235}, ISSN={["1090-2716"]}, DOI={10.1016/j.jcp.2012.11.006}, abstractNote={In this paper, we consider a practical filtering approach for assimilating irregularly spaced, sparsely observed turbulent signals through a hierarchical Bayesian reduced stochastic filtering framework. The proposed hierarchical Bayesian approach consists of two steps, blending a data-driven interpolation scheme and the Mean Stochastic Model (MSM) filter. We examine the potential of using the deterministic piecewise linear interpolation scheme and the ordinary kriging scheme in interpolating irregularly spaced raw data to regularly spaced processed data and the importance of dynamical constraint (through MSM) in filtering the processed data on a numerically stiff state estimation problem. In particular, we test this approach on a two-layer quasi-geostrophic model in a two-dimensional domain with a small radius of deformation to mimic ocean turbulence. Our numerical results suggest that the dynamical constraint becomes important when the observation noise variance is large. Second, we find that the filtered estimates with ordinary kriging are superior to those with linear interpolation when observation networks are not too sparse; such robust results are found from numerical simulations with many randomly simulated irregularly spaced observation networks, various observation time intervals, and observation error variances. Third, when the observation network is very sparse, we find that both the kriging and linear interpolations are comparable.}, journal={JOURNAL OF COMPUTATIONAL PHYSICS}, author={Brown, Kristen A. and Harlim, John}, year={2013}, month={Feb}, pages={143–160} }
 @article{bakunova_harlim_2013, title={Optimal filtering of complex turbulent systems with memory depth through consistency constraints}, volume={237}, ISSN={["1090-2716"]}, DOI={10.1016/j.jcp.2012.11.028}, abstractNote={In this article, we develop a linear theory for optimal filtering of complex turbulent signals with model errors through linear autoregressive models. We will show that when the autoregressive model parameters are chosen such that they satisfy absolute stability and consistency conditions of at least order-2 of the classical multistep method for solving initial value problems, the filtered solutions with autoregressive models of order p⩾2 are optimal in the sense that they are comparable to the estimates obtained from the true filter with perfect model. This result is reminiscent of the Lax-equivalence fundamental theorem in the analysis of finite difference discretization scheme for the numerical solutions of partial differential equations. We will apply this linear theory to filter two nonlinear problems, the slowest mode of the truncated Burgers–Hopf and the Lorenz-96 model. On these nonlinear problems, we will show that whenever these linear conditions are satisfied, the filtered solutions accuracy is significantly improved. Finally, we will also apply the recently developed offline test criteria to understand the robustness of the multistep filter on various turbulent nature, including the stochastically forced linear advection–diffusion equation and a toy model for barotropic turbulent Rossby waves.}, journal={JOURNAL OF COMPUTATIONAL PHYSICS}, author={Bakunova, Eugenia S. and Harlim, John}, year={2013}, month={Mar}, pages={320–343} }
 @article{majda_harlim_2013, title={Physics constrained nonlinear regression models for time series}, volume={26}, ISSN={["1361-6544"]}, DOI={10.1088/0951-7715/26/1/201}, abstractNote={A central issue in contemporary science is the development of data driven statistical nonlinear dynamical models for time series of partial observations of nature or a complex physical model. It has been established recently that ad hoc quadratic multi-level regression (MLR) models can have finite-time blow up of statistical solutions and/or pathological behaviour of their invariant measure. Here a new class of physics constrained multi-level quadratic regression models are introduced, analysed and applied to build reduced stochastic models from data of nonlinear systems. These models have the advantages of incorporating memory effects in time as well as the nonlinear noise from energy conserving nonlinear interactions. The mathematical guidelines for the performance and behaviour of these physics constrained MLR models as well as filtering algorithms for their implementation are developed here. Data driven applications of these new multi-level nonlinear regression models are developed for test models involving a nonlinear oscillator with memory effects and the difficult test case of the truncated Burgers–Hopf model. These new physics constrained quadratic MLR models are proposed here as process models for Bayesian estimation through Markov chain Monte Carlo algorithms of low frequency behaviour in complex physical data.}, number={1}, journal={NONLINEARITY}, author={Majda, Andrew J. and Harlim, John}, year={2013}, month={Jan}, pages={201–217} }
 @article{kang_harlim_majda_2013, title={Regression models with memory for the linear response of turbulent dynamical systems}, volume={11}, DOI={10.4310/cms.2013.v11.n2.a8}, abstractNote={Calculating the statistical linear response of turbulent dynamical systems to the change in external forcing is a problem of wide contemporary interest. Here the authors apply linear regression models with memory, AR(p) models, to approximate this statistical linear response by directly fitting the autocorrelations of the underlying turbulent dynamical system without further computational experiments. For highly nontrivial energy conserving turbulent dynamical systems like the Kruskal-Zabusky (KZ) or Truncated Burgers-Hopf (TBH) models, these AR(p) models exactly recover the mean linear statistical response to the change in external forcing at all response times with negligible errors. For a forced turbulent dynamical system like the Lorenz-96 (L-96) model, these approximations have improved skill comparable to the mean response with the quasi-Gaussian approximation for weakly chaotic turbulent dynamical systems. These AR(p) models also give new insight into the memory depth of the mean linear response operator for turbulent dynamical systems.}, number={2}, journal={Communications in Mathematical Sciences}, author={Kang, E. L. and Harlim, J. and Majda, A. J.}, year={2013}, pages={481–498} }
 @article{harlim_majda_2013, title={TEST MODELS FOR FILTERING WITH SUPERPARAMETERIZATION}, volume={11}, ISSN={["1540-3467"]}, DOI={10.1137/120890594}, abstractNote={Superparameterization is a fast numerical algorithm to mitigate implicit scale separation of dynamical systems with large-scale, slowly varying “mean” and smaller-scale, rapidly fluctuating “eddy” term. The main idea of superparameterization is to embed parallel highly resolved simulations of small-scale eddies on each grid cell of coarsely resolved large-scale dynamics. In this paper, we study the effect of model errors in using superparameterization for filtering multiscale turbulent dynamical systems. In particular, we use a simple test model, designed to mimic typical multiscale turbulent dynamics with small-scale intermittencies without local statistical equilibriation conditional to the large-scale mean dynamics, and simultaneously force the large-scale dynamics through eddy flux terms. In this paper, we consider the Fourier domain Kalman filter for filtering regularly spaced sparse observations of the large-scale mean variables. We find high filtering and statistical prediction skill with superpara...}, number={1}, journal={MULTISCALE MODELING & SIMULATION}, author={Harlim, John and Majda, Andrew J.}, year={2013}, pages={282–308} }
 @article{harlim_majda_2013, title={Test models for filtering and prediction of moisture-coupled tropical waves}, volume={139}, ISSN={["1477-870X"]}, DOI={10.1002/qj.1956}, abstractNote={The filtering/data assimilation and prediction of moisture‐coupled tropical waves is a contemporary topic with significant implications for extended‐range forecasting. The development of efficient algorithms to capture such waves is limited by the unstable multiscale features of tropical convection which can organize large‐scale circulations and the sparse observations of the moisture‐coupled wave in both the horizontal and vertical. The approach proposed here is to address these difficult issues of data assimilation and prediction through a suite of analogue models which, despite their simplicity, capture key features of the observational record and physical processes in moisture‐coupled tropical waves. The analogue models emphasized here involve the multicloud convective parametrization based on three cloud types (congestus, deep, and stratiform) above the boundary layer. Two test examples involving an MJO‐like turbulent travelling wave and the initiation of a convectively coupled wave train are introduced to illustrate the approach. A suite of reduced filters with judicious model errors for data assimilation of sparse observations of tropical waves, based on linear stochastic models in a moisture‐coupled eigenmode basis is developed here and applied to the two test problems. Both the reduced filter and 3D‐Var with a full moist background covariance matrix can recover the unobserved troposphere humidity and precipitation rate; on the other hand, 3D‐Var with a dry background covariance matrix fails to recover these unobserved variables. The skill of the reduced filtering methods in recovering the unobserved precipitation, congestus, and stratiform heating rates as well as the front‐to‐rear tilt of the convectively coupled waves exhibits a subtle dependence on the sparse observation network and the observation time. Copyright © 2012 Royal Meteorological Society}, number={670}, journal={QUARTERLY JOURNAL OF THE ROYAL METEOROLOGICAL SOCIETY}, author={Harlim, John and Majda, Andrew J.}, year={2013}, month={Jan}, pages={119–136} }
 @article{gottwald_harlim_2013, title={The role of additive and multiplicative noise in filtering complex dynamical systems}, volume={469}, ISSN={["1471-2946"]}, DOI={10.1098/rspa.2013.0096}, abstractNote={Covariance inflation is an ad hoc treatment that is widely used in practical real-time data assimilation algorithms to mitigate covariance underestimation owing to model errors, nonlinearity, or/and, in the context of ensemble filters, insufficient ensemble size. In this paper, we systematically derive an effective ‘statistical’ inflation for filtering multi-scale dynamical systems with moderate scale gap, , to the case of no scale gap with , in the presence of model errors through reduced dynamics from rigorous stochastic subgrid-scale parametrizations. We will demonstrate that for linear problems, an effective covariance inflation is achieved by a systematically derived additive noise in the forecast model, producing superior filtering skill. For nonlinear problems, we will study an analytically solvable stochastic test model, mimicking turbulent signals in regimes ranging from a turbulent energy transfer range to a dissipative range to a laminar regime. In this context, we will show that multiplicative noise naturally arises in addition to additive noise in a reduced stochastic forecast model. Subsequently, we will show that a ‘statistical’ inflation factor that involves mean correction in addition to covariance inflation is necessary to achieve accurate filtering in the presence of intermittent instability in both the turbulent energy transfer range and the dissipative range.}, number={2155}, journal={PROCEEDINGS OF THE ROYAL SOCIETY A-MATHEMATICAL PHYSICAL AND ENGINEERING SCIENCES}, author={Gottwald, Georg A. and Harlim, John}, year={2013}, month={Jul} }
 @book{majda_harlim_2012, title={Filtering Complex Turbulent Systems}, DOI={10.1017/cbo9781139061308}, abstractNote={Preface 1. Introduction and overview: mathematical strategies for filtering turbulent systems Part I. Fundamentals: 2. Filtering a stochastic complex scalar: the prototype test problem 3. The Kalman filter for vector systems: reduced filters and a three-dimensional toy model 4. Continuous and discrete Fourier series and numerical discretization Part II. Mathematical Guidelines for Filtering Turbulent Signals: 5. Stochastic models for turbulence 6. Filtering turbulent signals: plentiful observations 7. Filtering turbulent signals: regularly spaced sparse observations 8. Filtering linear stochastic PDE models with instability and model error Part III. Filtering Turbulent Nonlinear Dynamical Systems: 9. Strategies for filtering nonlinear systems 10. Filtering prototype nonlinear slow-fast systems 11. Filtering turbulent nonlinear dynamical systems by finite ensemble methods 12. Filtering turbulent nonlinear dynamical systems by linear stochastic models 13. Stochastic parameterized extended Kalman filter for filtering turbulent signal with model error 14. Filtering turbulent tracers from partial observations: an exactly solvable test model 15. The search for efficient skilful particle filters for high dimensional turbulent dynamical systems References Index.}, journal={FILTERING COMPLEX TURBULENT SYSTEMS}, author={Majda, AJ and Harlim, J}, year={2012}, pages={1–357} }
 @article{kang_harlim_2012, title={Filtering Partially Observed Multiscale Systems with Heterogeneous Multiscale Methods-Based Reduced Climate Models}, volume={140}, ISSN={["1520-0493"]}, DOI={10.1175/mwr-d-10-05067.1}, abstractNote={AbstractThis paper presents a fast reduced filtering strategy for assimilating multiscale systems in the presence of observations of only the macroscopic (or large scale) variables. This reduced filtering strategy introduces model errors in estimating the prior forecast statistics through the (heterogeneous multiscale methods) HMM-based reduced climate model as an alternative to the standard expensive (direct numerical simulation) DNS-based fully resolved model. More importantly, this approach is not restricted to any analysis (or Bayesian updating) step from various ensemble-based filters. In a regime where there is a distinctive separation of scales, high filtering skill is obtained through applying the HMM alone with any desirable analysis step from ensemble Kalman filters. When separation of scales is not apparent as typically observed in geophysical turbulent systems, an additional procedure is proposed to reinitialize the microscopic variables to statistically reflect pseudo-observations that are co...}, number={3}, journal={MONTHLY WEATHER REVIEW}, author={Kang, Emily L. and Harlim, John}, year={2012}, month={Mar}, pages={860–873} }
 @article{kang_harlim_2012, title={Filtering nonlinear spatio-temporal chaos with autoregressive linear stochastic models}, volume={241}, ISSN={["1872-8022"]}, DOI={10.1016/j.physd.2012.03.003}, abstractNote={Fundamental barriers in practical filtering of nonlinear spatio-temporal chaotic systems are model errors attributed to the stiffness in resolving multiscale features. Recently, reduced stochastic filters based on linear stochastic models have been introduced to overcome such stiffness; one of them is the Mean Stochastic Model (MSM) based on a diagonal Ornstein–Uhlenbeck process in Fourier space. Despite model errors, the MSM shows very encouraging filtering skill, especially when the hidden signal of interest is strongly chaotic. In this regime, the dynamical system statistical properties resemble to those of the energy-conserving equilibrium statistical mechanics with Gaussian invariant measure; therefore, the Ornstein–Uhlenbeck process with appropriate parameters is sufficient to produce reasonable statistical estimates for the filter model. In this paper, we consider a generalization of the MSM with a diagonal autoregressive linear stochastic model in Fourier space as a filter model for chaotic signals with long memory depth. With this generalization, the filter prior model becomes slightly more expensive than the MSM, but it is still less expensive relative to integrating the perfect model which is typically unknown in real problems. Furthermore, the associated Kalman filter on each Fourier mode is computationally as cheap as inverting a matrix of size D, where D is the number of observed variables on each Fourier mode (in our numerical example, D=1). Using the Lorenz 96 (L-96) model as a testbed, we show that the non-Markovian nature of this autoregressive model is an important feature in capturing the highly oscillatory modes with long memory depth. Second, we show that the filtering skill with autoregressive models supersedes that with MSM in weakly chaotic regime where the memory depth is longer. In strongly chaotic regime, the performance of the AR(p) filter is still better or at least comparable to that of the MSM. Most importantly, we find that this reduced filtering strategy is not as sensitive as standard ensemble filtering strategies to additional intrinsic model errors that are often encountered when model parameters are incorrectly specified.}, number={12}, journal={PHYSICA D-NONLINEAR PHENOMENA}, author={Kang, Emily L. and Harlim, John}, year={2012}, month={Jun}, pages={1099–1113} }
 @article{harlim_2011, title={INTERPOLATING IRREGULARLY SPACED OBSERVATIONS FOR FILTERING TURBULENT COMPLEX SYSTEMS}, volume={33}, ISSN={["1095-7197"]}, DOI={10.1137/100800427}, abstractNote={We present a numerically fast reduced filtering strategy, the Fourier domain Kalman filter with appropriate interpolations to account for irregularly spaced observations of complex turbulent signals. The design of such a reduced filter involves: (i) interpolating irregularly spaced observations to the model regularly spaced grid points, (ii) understanding under which situation the small scale oscillatory artifact from such interpolation won't degrade the filtered solutions, (iii) understanding when the interpolated covariance structure can be approximated by its diagonal terms when observations are corrupted by independent Gaussian noise, and (iv) applying a scalar Kalman filter formula on each Fourier component independently with an approximate diagonal interpolated covariance matrix. From the practical point of view, there is an emerging need to understand the effect of (i) toward the filtered solutions, for example, in utilizing the data produced from various interpolation techniques that merge multiple satellite measurements of atmospheric and ocean dynamical quantities. To understand point (iii) above and to see how many of nondiagonal terms are effectively ignored in (iv), we compute a ratio $\Lambda$ between the largest nondiagonal components and the smallest diagonal components of the interpolated covariance matrix. We find that for piecewise linear interpolation, this ratio, $\Lambda$, is always smaller than that of the alternative interpolation schemes such as trigonometric and cubic spline for any irregularly spaced observation networks. When observations are not so sparse with small noise, we find that the small scale oscillatory artifact in (ii) above is negligible when piecewise linear interpolation is used whereas for the other schemes such as the nearest neighbor, trigonometric, and cubic spline interpolation, the oscillatory artifact degrades the filtered solutions significantly. Finally, we also find that the reduced filtering strategy with piecewise linear interpolation produces more accurate filtered solutions than conventional approaches when observations are extremely irregularly spaced (such that the ratio $\Lambda$ is not so small) and very sparse.}, number={5}, journal={SIAM JOURNAL ON SCIENTIFIC COMPUTING}, author={Harlim, John}, year={2011}, pages={2620–2640} }
 @article{harlim_2011, title={Numerical strategies for filtering partially observed stiff stochastic differential equations}, volume={230}, ISSN={["1090-2716"]}, DOI={10.1016/j.jcp.2010.10.016}, abstractNote={In this paper, we present a fast numerical strategy for filtering stochastic differential equations with multiscale features. This method is designed such that it does not violate the practical linear observability condition and, more importantly, it does not require the computationally expensive cross correlation statistics between multiscale variables that are typically needed in standard filtering approach. The proposed filtering algorithm comprises of a “macro-filter” that borrows ideas from the Heterogeneous Multiscale Methods and a “micro-filter” that reinitializes the fast microscopic variables to statistically reflect the unbiased slow macroscopic estimate obtained from the macro-filter and macroscopic observations at asynchronous times. We will show that the proposed micro-filter is equivalent to solving an inverse problem for parameterizing differential equations. Numerically, we will show that this microscopic reinitialization is an important novel feature for accurate filtered solutions, especially when the microscopic dynamics is not mixing at all.}, number={3}, journal={JOURNAL OF COMPUTATIONAL PHYSICS}, author={Harlim, John}, year={2011}, month={Feb}, pages={744–762} }
 @article{harlim_majda_2010, title={Filtering Turbulent Sparsely Observed Geophysical Flows}, volume={138}, ISSN={["1520-0493"]}, DOI={10.1175/2009mwr3113.1}, abstractNote={Abstract Filtering sparsely turbulent signals from nature is a central problem of contemporary data assimilation. Here, sparsely observed turbulent signals from nature are generated by solutions of two-layer quasigeostrophic models with turbulent cascades from baroclinic instability in two separate regimes with varying Rossby radius mimicking the atmosphere and the ocean. In the “atmospheric” case, large-scale turbulent fluctuations are dominated by barotropic zonal jets with non-Gaussian statistics while the “oceanic” case has large-scale blocking regime transitions with barotropic zonal jets and large-scale Rossby waves. Recently introduced, cheap radical linear stochastic filtering algorithms utilizing mean stochastic models (MSM1, MSM2) that have judicious model errors are developed here as well as a very recent cheap stochastic parameterization extended Kalman filter (SPEKF), which includes stochastic parameterization of additive and multiplicative bias corrections “on the fly.” These cheap stochasti...}, number={4}, journal={MONTHLY WEATHER REVIEW}, author={Harlim, John and Majda, Andrew J.}, year={2010}, month={Apr}, pages={1050–1083} }
 @article{gershgorin_harlim_majda_2010, title={Improving filtering and prediction of spatially extended turbulent systems with model errors through stochastic parameter estimation}, volume={229}, ISSN={["1090-2716"]}, DOI={10.1016/j.jcp.2009.09.022}, abstractNote={The filtering and predictive skill for turbulent signals is often limited by the lack of information about the true dynamics of the system and by our inability to resolve the assumed dynamics with sufficiently high resolution using the current computing power. The standard approach is to use a simple yet rich family of constant parameters to account for model errors through parameterization. This approach can have significant skill by fitting the parameters to some statistical feature of the true signal; however in the context of real-time prediction, such a strategy performs poorly when intermittent transitions to instability occur. Alternatively, we need a set of dynamic parameters. One strategy for estimating parameters on the fly is a stochastic parameter estimation through partial observations of the true signal. In this paper, we extend our newly developed stochastic parameter estimation strategy, the Stochastic Parameterization Extended Kalman Filter (SPEKF), to filtering sparsely observed spatially extended turbulent systems which exhibit abrupt stability transition from time to time despite a stable average behavior. For our primary numerical example, we consider a turbulent system of externally forced barotropic Rossby waves with instability introduced through intermittent negative damping. We find high filtering skill of SPEKF applied to this toy model even in the case of very sparse observations (with only 15 out of the 105 grid points observed) and with unspecified external forcing and damping. Additive and multiplicative bias corrections are used to learn the unknown features of the true dynamics from observations. We also present a comprehensive study of predictive skill in the one-mode context including the robustness toward variation of stochastic parameters, imperfect initial conditions and finite ensemble effect. Furthermore, the proposed stochastic parameter estimation scheme applied to the same spatially extended Rossby wave system demonstrates high predictive skill, comparable with the skill of the perfect model for a duration of many eddy turnover times especially in the unstable regime.}, number={1}, journal={JOURNAL OF COMPUTATIONAL PHYSICS}, author={Gershgorin, B. and Harlim, J. and Majda, A. J.}, year={2010}, month={Jan}, pages={32–57} }
 @article{majda_harlim_gershgorin_2010, title={MATHEMATICAL STRATEGIES FOR FILTERING TURBULENT DYNAMICAL SYSTEMS}, volume={27}, ISSN={["1553-5231"]}, DOI={10.3934/dcds.2010.27.441}, abstractNote={The modus operandi of modern applied mathematics in developing very recent mathematical strategies for filtering turbulent dynamical systems is 
emphasized here. The approach involves the synergy of rigorous mathematical guidelines, exactly solvable nonlinear models with physical insight, 
and novel cheap algorithms with judicious model errors to filter turbulent signals with many degrees of freedom. A large number of new theoretical and 
computational phenomena such as "catastrophic filter divergence" in finite ensemble filters are reviewed here with the intention to introduce mathematicians, 
applied mathematicians, and scientists to this remarkable emerging scientific discipline with increasing practical importance.}, number={2}, journal={DISCRETE AND CONTINUOUS DYNAMICAL SYSTEMS}, author={Majda, Andrew J. and Harlim, John and Gershgorin, Boris}, year={2010}, month={Jun}, pages={441–486} }
 @article{gershgorin_harlim_majda_2010, title={Test models for improving filtering with model errors through stochastic parameter estimation}, volume={229}, ISSN={["1090-2716"]}, DOI={10.1016/j.jcp.2009.08.019}, abstractNote={The filtering skill for turbulent signals from nature is often limited by model errors created by utilizing an imperfect model for filtering. Updating the parameters in the imperfect model through stochastic parameter estimation is one way to increase filtering skill and model performance. Here a suite of stringent test models for filtering with stochastic parameter estimation is developed based on the Stochastic Parameterization Extended Kalman Filter (SPEKF). These new SPEKF-algorithms systematically correct both multiplicative and additive biases and involve exact formulas for propagating the mean and covariance including the parameters in the test model. A comprehensive study is presented of robust parameter regimes for increasing filtering skill through stochastic parameter estimation for turbulent signals as the observation time and observation noise are varied and even when the forcing is incorrectly specified. The results here provide useful guidelines for filtering turbulent signals in more complex systems with significant model errors.}, number={1}, journal={JOURNAL OF COMPUTATIONAL PHYSICS}, author={Gershgorin, B. and Harlim, J. and Majda, A. J.}, year={2010}, month={Jan}, pages={1–31} }