{"id":{"repo_id":"iastate","oai_identifier":"oai:dr.lib.iastate.edu:20.500.12876/YvkAO7Bz"},"canonical_url":"https://search.dev.ndltd.org/etd/iastate/oai:dr.lib.iastate.edu:20.500.12876/YvkAO7Bz","repository":{"repo_id":"iastate","name":"Iowa State University","base_url":"https://dr.lib.iastate.edu/server/oai/request"},"display":{"title":"Addressing stale gradients in asynchronous federated deep reinforcement learning","abstract":"Advancements in reinforcement learning (RL) via deep neural networks have enabled their application to a variety of real-world problems. However, these applications often suffer from long training times. While attempts to distribute training have been successful in controlled scenarios, they face challenges in heterogeneous-capacity, unstable, and privacy critical environments. This work applies concepts from federated learning (FL) to distributed RL, specifically addressing the stale gradient problem. A deterministic framework for asynchronous federated RL is utilized to explore dynamic methods for handling stale gradient updates in the Arcade Learning Environment. Experimental results from applying these methods to two Atari-2600 games demonstrate a relative speedup of up to 95\\% compared to plain A3C in large and unstable federations.","abstract_html":"Advancements in reinforcement learning (RL) via deep neural networks have enabled their application to a variety of real-world problems. However, these applications often suffer from long training times. While attempts to distribute training have been successful in controlled scenarios, they face challenges in heterogeneous-capacity, unstable, and privacy critical environments. This work applies concepts from federated learning (FL) to distributed RL, specifically addressing the stale gradient problem. A deterministic framework for asynchronous federated RL is utilized to explore dynamic methods for handling stale gradient updates in the Arcade Learning Environment. Experimental results from applying these methods to two Atari-2600 games demonstrate a relative speedup of up to 95\\% compared to plain A3C in large and unstable federations.","abstract_has_math":false,"creators":["Stanley, Justin"],"institution":"Iowa State University","degree_name":"Master of Science","degree_level":"thesis","degree_discipline":"Computer science","degree_department":"Department of Computer Science","school":null,"contributors":[],"advisors":["Jannesari, Ali","Quinn, Christopher","Tian, Jin","Huai, Mengdi"],"committee_chairs":[],"committee_members":[],"year":2023,"date_issued":"2023-08","date_published":"2023-08","updated_at":"2026-07-24T02:38:01Z","subjects":[],"languages":["en"],"rights":[],"rights_urls":[],"identifier_entries":[{"key":"dc:identifier.doi","label":"DOI","values":["https://doi.org/10.31274/td-20240329-43"],"render_values":[{"text":"https://doi.org/10.31274/td-20240329-43","href":"https://doi.org/10.31274/td-20240329-43","code":true}]}]},"links":{"outbound_url":"https://dr.lib.iastate.edu/handle/20.500.12876/YvkAO7Bz","outbound_label":"Repository record","outbound_source":"dc:identifier.uri"},"metadata_groups":[{"id":"people","label":"People","entries":[{"key":"dc:contributor.advisor","label":"Advisor","values":["Jannesari, Ali","Quinn, Christopher","Tian, Jin","Huai, Mengdi"]},{"key":"dc:contributor.department","label":"Department","values":["Department of Computer Science"]},{"key":"dc:creator","label":"Author","values":["Stanley, Justin"]}]},{"id":"academic_context","label":"Academic Context","entries":[{"key":"dc:date.accessioned","label":"Dc Date Accessioned","values":["2023-09-18T04:24:57Z"]},{"key":"dc:date.available","label":"Dc Date Available","values":["2023-09-18T04:24:57Z"]},{"key":"dc:date.issued","label":"Date","values":["2023-08"]},{"key":"dc:type","label":"Dc Type","values":["thesis"]},{"key":"thesis:degree_discipline","label":"Discipline","values":["Computer science"]},{"key":"thesis:degree_level","label":"Degree Level","values":["thesis"]},{"key":"thesis:degree_name","label":"Degree Name","values":["Master of Science"]},{"key":"thesis:institution_name","label":"Thesis Institution Name","values":["Iowa State University"]}]},{"id":"language_rights","label":"Language and Rights","entries":[{"key":"dc:language.iso","label":"Language (ISO)","values":["en"]}]},{"id":"identifiers","label":"Identifiers","entries":[{"key":"dc:identifier.doi","label":"DOI","values":["https://doi.org/10.31274/td-20240329-43"]},{"key":"dc:identifier.uri","label":"Identifier URI","values":["https://dr.lib.iastate.edu/handle/20.500.12876/YvkAO7Bz"]}]},{"id":"additional","label":"Additional Metadata","entries":[{"key":"dc:description.abstract","label":"Abstract","values":["Advancements in reinforcement learning (RL) via deep neural networks have enabled their application to a variety of real-world problems. However, these applications often suffer from long training times. While attempts to distribute training have been successful in controlled scenarios, they face challenges in heterogeneous-capacity, unstable, and privacy critical environments. This work applies concepts from federated learning (FL) to distributed RL, specifically addressing the stale gradient problem. A deterministic framework for asynchronous federated RL is utilized to explore dynamic methods for handling stale gradient updates in the Arcade Learning Environment. Experimental results from applying these methods to two Atari-2600 games demonstrate a relative speedup of up to 95\\% compared to plain A3C in large and unstable federations."]},{"key":"dc:format.mimetype","label":"Dc Format Mimetype","values":["PDF"]},{"key":"dc:title","label":"Title","values":["Addressing stale gradients in asynchronous federated deep reinforcement learning"]}]}],"canonical_facts":{"dc:contributor.advisor":["Jannesari, Ali","Quinn, Christopher","Tian, Jin","Huai, Mengdi"],"dc:contributor.department":["Department of Computer Science"],"dc:creator":["Stanley, Justin"],"dc:date.accessioned":["2023-09-18T04:24:57Z"],"dc:date.available":["2023-09-18T04:24:57Z"],"dc:date.issued":["2023-08"],"dc:description.abstract":["Advancements in reinforcement learning (RL) via deep neural networks have enabled their application to a variety of real-world problems. However, these applications often suffer from long training times. While attempts to distribute training have been successful in controlled scenarios, they face challenges in heterogeneous-capacity, unstable, and privacy critical environments. This work applies concepts from federated learning (FL) to distributed RL, specifically addressing the stale gradient problem. A deterministic framework for asynchronous federated RL is utilized to explore dynamic methods for handling stale gradient updates in the Arcade Learning Environment. Experimental results from applying these methods to two Atari-2600 games demonstrate a relative speedup of up to 95\\% compared to plain A3C in large and unstable federations."],"dc:format.mimetype":["PDF"],"dc:identifier.doi":["https://doi.org/10.31274/td-20240329-43"],"dc:identifier.uri":["https://dr.lib.iastate.edu/handle/20.500.12876/YvkAO7Bz"],"dc:language.iso":["en"],"dc:title":["Addressing stale gradients in asynchronous federated deep reinforcement learning"],"dc:type":["thesis"],"thesis:degree_discipline":["Computer science"],"thesis:degree_level":["thesis"],"thesis:degree_name":["Master of Science"],"thesis:institution_name":["Iowa State University"]},"updated_at":"2026-07-24T02:38:01Z"}