{"id":{"repo_id":"nus","oai_identifier":"oai:scholarbank.nus.edu.sg:10635/309571"},"canonical_url":"https://search.dev.ndltd.org/etd/nus/oai:scholarbank.nus.edu.sg:10635/309571","repository":{"repo_id":"nus","name":"National University of Singapore","base_url":"https://scholarbank.nus.edu.sg/oai/request"},"display":{"title":"INVERSE APPROXIMATION THEORY OF RECURRENT MODELS FOR LEARNING SEQUENCES","abstract":"Learning long-term relationships is a challenging task in sequence modelling. Despite numerous empirical results demonstrating the difficulty of recurrent models in learning long-term relationships, this dissertation presents a series of theoretical studies on the learning of long-term memories using recurrent models. First, based on the concept of a generalized memory function over nonlinear functional sequences, we prove that adding nonlinear activation does not change the asymptotic exponential memory decay pattern. Then we prove the universal approximation property for state-space models. A similar inverse approximation result is established for state-space models, indicating that despite their high efficiency, changing the method of incorporating nonlinear activation fails to alleviate the memory decay constraint observed in the first part. Based on the proofs, we identify that suitable reparameterizations are the key to stably approximation. A class of stable reparameterizations enables state-space models to achieve stable approximation for any target with decaying memory.","abstract_html":"Learning long-term relationships is a challenging task in sequence modelling. Despite numerous empirical results demonstrating the difficulty of recurrent models in learning long-term relationships, this dissertation presents a series of theoretical studies on the learning of long-term memories using recurrent models. First, based on the concept of a generalized memory function over nonlinear functional sequences, we prove that adding nonlinear activation does not change the asymptotic exponential memory decay pattern. Then we prove the universal approximation property for state-space models. A similar inverse approximation result is established for state-space models, indicating that despite their high efficiency, changing the method of incorporating nonlinear activation fails to alleviate the memory decay constraint observed in the first part. Based on the proofs, we identify that suitable reparameterizations are the key to stably approximation. A class of stable reparameterizations enables state-space models to achieve stable approximation for any target with decaying memory.","abstract_has_math":false,"creators":["WANG SHIDA"],"institution":null,"degree_name":null,"degree_level":null,"degree_discipline":null,"degree_department":null,"school":null,"contributors":[],"advisors":[],"committee_chairs":[],"committee_members":[],"year":2024,"date_issued":"2024-07-15","date_published":"2024-07-15","updated_at":"2026-07-24T03:33:09Z","subjects":["memory function","stable approximation","state-space model","recurrent model","sequence modelling","Inverse approximation theory"],"languages":[],"rights":[],"rights_urls":["https://scholarbank.nus.edu.sg/bitstreams/1387f33c-4aca-4167-b20a-4850d030558f/download"],"identifier_entries":[]},"links":{"outbound_url":null,"outbound_label":null,"outbound_source":null},"metadata_groups":[{"id":"people","label":"People","entries":[{"key":"dc:creator","label":"Author","values":["WANG SHIDA"]}]},{"id":"academic_context","label":"Academic Context","entries":[{"key":"dc:date.issued","label":"Date","values":["2024-07-15"]},{"key":"dc:relation.isreferencedby","label":"Dc Relation Isreferencedby","values":["https://scholarbank.nus.edu.sg/handle/10635/309571"]},{"key":"dc:type","label":"Dc Type","values":["Thesis"]}]},{"id":"subjects_keywords","label":"Subjects and Keywords","entries":[{"key":"dc:subject","label":"Dc Subject","values":["memory function","stable approximation","state-space model","recurrent model","sequence modelling","Inverse approximation theory"]}]},{"id":"language_rights","label":"Language and Rights","entries":[{"key":"dc:rights","label":"Dc Rights","values":["https://scholarbank.nus.edu.sg/bitstreams/1387f33c-4aca-4167-b20a-4850d030558f/download"]}]},{"id":"identifiers","label":"Identifiers","entries":[{"key":"dc:identifier.uri","label":"Identifier URI","values":["https://scholarbank.nus.edu.sg/bitstreams/792b1124-f035-44fb-91e4-e1f04af352bf/download"]}]},{"id":"additional","label":"Additional Metadata","entries":[{"key":"dc:description.abstract","label":"Abstract","values":["Learning long-term relationships is a challenging task in sequence modelling. Despite numerous empirical results demonstrating the difficulty of recurrent models in learning long-term relationships, this dissertation presents a series of theoretical studies on the learning of long-term memories using recurrent models. First, based on the concept of a generalized memory function over nonlinear functional sequences, we prove that adding nonlinear activation does not change the asymptotic exponential memory decay pattern. Then we prove the universal approximation property for state-space models. A similar inverse approximation result is established for state-space models, indicating that despite their high efficiency, changing the method of incorporating nonlinear activation fails to alleviate the memory decay constraint observed in the first part. Based on the proofs, we identify that suitable reparameterizations are the key to stably approximation. A class of stable reparameterizations enables state-space models to achieve stable approximation for any target with decaying memory."]},{"key":"dc:format.checksum.md5","label":"Dc Format Checksum Md5","values":["9f3c6f2aa8f96291ef77b0a18484521d","941e23517a28e2487eb398db3ba32098","de24bec0b0ca0cb65a74670a9f018351"]},{"key":"dc:title","label":"Title","values":["INVERSE APPROXIMATION THEORY OF RECURRENT MODELS FOR LEARNING SEQUENCES"]}]}],"canonical_facts":{"dc:creator":["WANG SHIDA"],"dc:date.issued":["2024-07-15"],"dc:description.abstract":["Learning long-term relationships is a challenging task in sequence modelling. Despite numerous empirical results demonstrating the difficulty of recurrent models in learning long-term relationships, this dissertation presents a series of theoretical studies on the learning of long-term memories using recurrent models. First, based on the concept of a generalized memory function over nonlinear functional sequences, we prove that adding nonlinear activation does not change the asymptotic exponential memory decay pattern. Then we prove the universal approximation property for state-space models. A similar inverse approximation result is established for state-space models, indicating that despite their high efficiency, changing the method of incorporating nonlinear activation fails to alleviate the memory decay constraint observed in the first part. Based on the proofs, we identify that suitable reparameterizations are the key to stably approximation. A class of stable reparameterizations enables state-space models to achieve stable approximation for any target with decaying memory."],"dc:format.checksum.md5":["9f3c6f2aa8f96291ef77b0a18484521d","941e23517a28e2487eb398db3ba32098","de24bec0b0ca0cb65a74670a9f018351"],"dc:identifier.uri":["https://scholarbank.nus.edu.sg/bitstreams/792b1124-f035-44fb-91e4-e1f04af352bf/download"],"dc:relation.isreferencedby":["https://scholarbank.nus.edu.sg/handle/10635/309571"],"dc:rights":["https://scholarbank.nus.edu.sg/bitstreams/1387f33c-4aca-4167-b20a-4850d030558f/download"],"dc:subject":["memory function","stable approximation","state-space model","recurrent model","sequence modelling","Inverse approximation theory"],"dc:title":["INVERSE APPROXIMATION THEORY OF RECURRENT MODELS FOR LEARNING SEQUENCES"],"dc:type":["Thesis"]},"updated_at":"2026-07-24T03:33:09Z"}