{"id":{"repo_id":"mit","oai_identifier":"oai:dspace.mit.edu:1721.1/139564"},"canonical_url":"https://search.dev.ndltd.org/etd/mit/oai:dspace.mit.edu:1721.1/139564","repository":{"repo_id":"mit","name":"MIT","base_url":"https://dspace.mit.edu/oai/request"},"display":{"title":"Understanding Vision-based Dynamics Models","abstract":"Recent developments in vision-based dynamics models have helped researchers achieve state-of-the-art results in a number of fields. For instance, in model-based reinforcement learning, vision-based methods perform extremely well on a variety of games and control tasks while using orders of magnitudes less data than model-free methods. One example is GameGAN, which learns to simulate the dynamics of observed games solely from visual and action inputs. However, there is very little understanding of these models and how they work. To address this lack of understanding, we apply the Network Dissection framework to analyze vision-based dynamics prediction models. We inspect individual trained neurons in convolutional layers of these models and modify the output of neurons to understand their effect on the representation. We also theoretically extend the Network Dissection framework by generalizing it to fully connected layers instead of only convolutional layers. Overall, we provide insight into the node-level workings of dynamics models.","abstract_html":"Recent developments in vision-based dynamics models have helped researchers achieve state-of-the-art results in a number of fields. For instance, in model-based reinforcement learning, vision-based methods perform extremely well on a variety of games and control tasks while using orders of magnitudes less data than model-free methods. One example is GameGAN, which learns to simulate the dynamics of observed games solely from visual and action inputs. However, there is very little understanding of these models and how they work. To address this lack of understanding, we apply the Network Dissection framework to analyze vision-based dynamics prediction models. We inspect individual trained neurons in convolutional layers of these models and modify the output of neurons to understand their effect on the representation. We also theoretically extend the Network Dissection framework by generalizing it to fully connected layers instead of only convolutional layers. Overall, we provide insight into the node-level workings of dynamics models.","abstract_has_math":false,"creators":["Liu, Cynthia"],"institution":"Massachusetts Institute of Technology","degree_name":"Master","degree_level":null,"degree_discipline":null,"degree_department":"Massachusetts Institute of Technology. Department of Electrical Engineering and Computer Science","school":null,"contributors":[],"advisors":["Torralba, Antonio"],"committee_chairs":[],"committee_members":[],"year":2021,"date_issued":"2021-06","date_published":"2021-06","updated_at":"2026-07-22T22:22:13Z","subjects":[],"languages":[],"rights":["In Copyright - Educational Use Permitted","Copyright MIT"],"rights_urls":["http://rightsstatements.org/page/InC-EDU/1.0/"],"identifier_entries":[]},"links":{"outbound_url":"https://hdl.handle.net/1721.1/139564","outbound_label":"Handle","outbound_source":"dc:identifier.uri"},"metadata_groups":[{"id":"people","label":"People","entries":[{"key":"dc:contributor.advisor","label":"Advisor","values":["Torralba, Antonio"]},{"key":"dc:contributor.department","label":"Department","values":["Massachusetts Institute of Technology. Department of Electrical Engineering and Computer Science"]},{"key":"dc:creator","label":"Author","values":["Liu, Cynthia"]}]},{"id":"academic_context","label":"Academic Context","entries":[{"key":"dc:date.accessioned","label":"Dc Date Accessioned","values":["2022-01-14T15:20:14Z"]},{"key":"dc:date.available","label":"Dc Date Available","values":["2022-01-14T15:20:14Z"]},{"key":"dc:date.issued","label":"Date","values":["2021-06"]},{"key":"dc:publisher","label":"Institution","values":["Massachusetts Institute of Technology"]},{"key":"dc:type","label":"Dc Type","values":["Thesis"]},{"key":"thesis:degree_name","label":"Degree Name","values":["Master","Master of Engineering in Electrical Engineering and Computer Science"]}]},{"id":"language_rights","label":"Language and Rights","entries":[{"key":"dc:rights","label":"Dc Rights","values":["In Copyright - Educational Use Permitted","Copyright MIT"]},{"key":"dc:rights.uri","label":"Rights URI","values":["http://rightsstatements.org/page/InC-EDU/1.0/"]}]},{"id":"identifiers","label":"Identifiers","entries":[{"key":"dc:identifier.uri","label":"Identifier URI","values":["https://hdl.handle.net/1721.1/139564"]}]},{"id":"additional","label":"Additional Metadata","entries":[{"key":"dc:description.abstract","label":"Abstract","values":["Recent developments in vision-based dynamics models have helped researchers achieve state-of-the-art results in a number of fields. For instance, in model-based reinforcement learning, vision-based methods perform extremely well on a variety of games and control tasks while using orders of magnitudes less data than model-free methods. One example is GameGAN, which learns to simulate the dynamics of observed games solely from visual and action inputs. However, there is very little understanding of these models and how they work. To address this lack of understanding, we apply the Network Dissection framework to analyze vision-based dynamics prediction models. We inspect individual trained neurons in convolutional layers of these models and modify the output of neurons to understand their effect on the representation. We also theoretically extend the Network Dissection framework by generalizing it to fully connected layers instead of only convolutional layers. Overall, we provide insight into the node-level workings of dynamics models."]},{"key":"dc:description.degree","label":"Dc Description Degree","values":["M.Eng."]},{"key":"dc:title","label":"Title","values":["Understanding Vision-based Dynamics Models"]}]}],"canonical_facts":{"dc:contributor.advisor":["Torralba, Antonio"],"dc:contributor.department":["Massachusetts Institute of Technology. Department of Electrical Engineering and Computer Science"],"dc:creator":["Liu, Cynthia"],"dc:date.accessioned":["2022-01-14T15:20:14Z"],"dc:date.available":["2022-01-14T15:20:14Z"],"dc:date.issued":["2021-06"],"dc:description.abstract":["Recent developments in vision-based dynamics models have helped researchers achieve state-of-the-art results in a number of fields. For instance, in model-based reinforcement learning, vision-based methods perform extremely well on a variety of games and control tasks while using orders of magnitudes less data than model-free methods. One example is GameGAN, which learns to simulate the dynamics of observed games solely from visual and action inputs. However, there is very little understanding of these models and how they work. To address this lack of understanding, we apply the Network Dissection framework to analyze vision-based dynamics prediction models. We inspect individual trained neurons in convolutional layers of these models and modify the output of neurons to understand their effect on the representation. We also theoretically extend the Network Dissection framework by generalizing it to fully connected layers instead of only convolutional layers. Overall, we provide insight into the node-level workings of dynamics models."],"dc:description.degree":["M.Eng."],"dc:identifier.uri":["https://hdl.handle.net/1721.1/139564"],"dc:publisher":["Massachusetts Institute of Technology"],"dc:rights":["In Copyright - Educational Use Permitted","Copyright MIT"],"dc:rights.uri":["http://rightsstatements.org/page/InC-EDU/1.0/"],"dc:title":["Understanding Vision-based Dynamics Models"],"dc:type":["Thesis"],"thesis:degree_name":["Master","Master of Engineering in Electrical Engineering and Computer Science"]},"updated_at":"2026-07-22T22:22:13Z"}