{"id":{"repo_id":"nus","oai_identifier":"oai:scholarbank.nus.edu.sg:10635/166275"},"canonical_url":"https://search.dev.ndltd.org/etd/nus/oai:scholarbank.nus.edu.sg:10635/166275","repository":{"repo_id":"nus","name":"National University of Singapore","base_url":"https://scholarbank.nus.edu.sg/oai/request"},"display":{"title":"REGULARIZATION ON MACHINE LEARNING","abstract":"Deep neural networks have become a powerful tool for machine learning problems. However, overfitting frequently occurs. To achieve better generalization, many regularization methods were proposed to reduce overfitting. In this thesis, we propose a simple-yet-effective regularization method called Drop-Activation. At the training phase, we drop nonlinear activation functions randomly and set them to be identity functions. At the testing phase, a deterministic network with a new activation function is used and the new activation function is designed to average effect of the randomness of discarding activations. We theoretically deduce the implicit regularization terms of Drop-Activation and the effect of Drop-Activation can be considered as implicit parameter reduction. Also, our theoretical analysis verifies its capability to be used together with Batch Normalization (Ioffe and Szegedy 2015). We perform Drop-Activation on the benchmark datasets and show that the performance of popular networks can be improved generally by Drop-Activation.","abstract_html":"Deep neural networks have become a powerful tool for machine learning problems. However, overfitting frequently occurs. To achieve better generalization, many regularization methods were proposed to reduce overfitting. In this thesis, we propose a simple-yet-effective regularization method called Drop-Activation. At the training phase, we drop nonlinear activation functions randomly and set them to be identity functions. At the testing phase, a deterministic network with a new activation function is used and the new activation function is designed to average effect of the randomness of discarding activations. We theoretically deduce the implicit regularization terms of Drop-Activation and the effect of Drop-Activation can be considered as implicit parameter reduction. Also, our theoretical analysis verifies its capability to be used together with Batch Normalization (Ioffe and Szegedy 2015). We perform Drop-Activation on the benchmark datasets and show that the performance of popular networks can be improved generally by Drop-Activation.","abstract_has_math":false,"creators":["LIANG SENWEI"],"institution":null,"degree_name":null,"degree_level":null,"degree_discipline":null,"degree_department":null,"school":null,"contributors":[],"advisors":["Yang Haizhao"],"committee_chairs":[],"committee_members":[],"year":2019,"date_issued":"2019-12-18","date_published":"2019-12-18","updated_at":"2026-08-21T16:47:09Z","subjects":["deep learning, regularization, generalization, overfitting, Drop-Activation"],"languages":["en"],"rights":[],"rights_urls":[],"identifier_entries":[]},"links":{"outbound_url":"https://scholarbank.nus.edu.sg/handle/10635/166275","outbound_label":"Repository record","outbound_source":"dc:identifier.uri"},"source_record":{"url":"https://scholarbank.nus.edu.sg/oai/request?verb=GetRecord&metadataPrefix=dim&identifier=oai%3Ascholarbank.nus.edu.sg%3A10635%2F166275","prefix":"dim"},"metadata_groups":[{"id":"people","label":"People","entries":[{"key":"dc:contributor.other","label":"Dc Contributor Other","values":["MATHEMATICS"]},{"key":"dc:contributor.supervisor","label":"Supervisor","values":["Yang Haizhao"]},{"key":"dc:creator","label":"Author","values":["LIANG SENWEI"]}]},{"id":"academic_context","label":"Academic Context","entries":[{"key":"dc:date.accessioned","label":"Dc Date Accessioned","values":["2020-03-31T18:00:51Z"]},{"key":"dc:date.available","label":"Dc Date Available","values":["2020-03-31T18:00:51Z"]},{"key":"dc:date.issued","label":"Date","values":["2019-12-18"]},{"key":"dc:relation.isreferencedby","label":"Dc Relation Isreferencedby","values":["https://scholarbank.nus.edu.sg/handle/10635/166275"]},{"key":"dc:type","label":"Dc Type","values":["Thesis"]}]},{"id":"subjects_keywords","label":"Subjects and Keywords","entries":[{"key":"dc:subject","label":"Dc Subject","values":["deep learning, regularization, generalization, overfitting, Drop-Activation"]}]},{"id":"language_rights","label":"Language and Rights","entries":[{"key":"dc:language.iso","label":"Language (ISO)","values":["en"]}]},{"id":"identifiers","label":"Identifiers","entries":[{"key":"dc:identifier.uri","label":"Identifier URI","values":["https://scholarbank.nus.edu.sg/bitstreams/3f2441a9-8bec-4f85-baf1-45e6c4294d27/download","https://scholarbank.nus.edu.sg/handle/10635/166275"]}]},{"id":"additional","label":"Additional Metadata","entries":[{"key":"dc:description.abstract","label":"Abstract","values":["Deep neural networks have become a powerful tool for machine learning problems. However, overfitting frequently occurs. To achieve better generalization, many regularization methods were proposed to reduce overfitting. In this thesis, we propose a simple-yet-effective regularization method called Drop-Activation. At the training phase, we drop nonlinear activation functions randomly and set them to be identity functions. At the testing phase, a deterministic network with a new activation function is used and the new activation function is designed to average effect of the randomness of discarding activations. We theoretically deduce the implicit regularization terms of Drop-Activation and the effect of Drop-Activation can be considered as implicit parameter reduction. Also, our theoretical analysis verifies its capability to be used together with Batch Normalization (Ioffe and Szegedy 2015). We perform Drop-Activation on the benchmark datasets and show that the performance of popular networks can be improved generally by Drop-Activation."]},{"key":"dc:description.degree","label":"Dc Description Degree","values":["Master's"]},{"key":"dc:format.checksum.md5","label":"Dc Format Checksum Md5","values":["97978810ddc46456f40627f6c35fea47","639d7e8f3157c6b35f1279cc9dff717e"]},{"key":"dc:title","label":"Title","values":["REGULARIZATION ON MACHINE LEARNING"]}]}],"canonical_facts":{"dc:contributor.other":["MATHEMATICS"],"dc:contributor.supervisor":["Yang Haizhao"],"dc:creator":["LIANG SENWEI"],"dc:date.accessioned":["2020-03-31T18:00:51Z"],"dc:date.available":["2020-03-31T18:00:51Z"],"dc:date.issued":["2019-12-18"],"dc:description.abstract":["Deep neural networks have become a powerful tool for machine learning problems. However, overfitting frequently occurs. To achieve better generalization, many regularization methods were proposed to reduce overfitting. In this thesis, we propose a simple-yet-effective regularization method called Drop-Activation. At the training phase, we drop nonlinear activation functions randomly and set them to be identity functions. At the testing phase, a deterministic network with a new activation function is used and the new activation function is designed to average effect of the randomness of discarding activations. We theoretically deduce the implicit regularization terms of Drop-Activation and the effect of Drop-Activation can be considered as implicit parameter reduction. Also, our theoretical analysis verifies its capability to be used together with Batch Normalization (Ioffe and Szegedy 2015). We perform Drop-Activation on the benchmark datasets and show that the performance of popular networks can be improved generally by Drop-Activation."],"dc:description.degree":["Master's"],"dc:format.checksum.md5":["97978810ddc46456f40627f6c35fea47","639d7e8f3157c6b35f1279cc9dff717e"],"dc:identifier.uri":["https://scholarbank.nus.edu.sg/bitstreams/3f2441a9-8bec-4f85-baf1-45e6c4294d27/download","https://scholarbank.nus.edu.sg/handle/10635/166275"],"dc:language.iso":["en"],"dc:relation.isreferencedby":["https://scholarbank.nus.edu.sg/handle/10635/166275"],"dc:subject":["deep learning, regularization, generalization, overfitting, Drop-Activation"],"dc:title":["REGULARIZATION ON MACHINE LEARNING"],"dc:type":["Thesis"]},"updated_at":"2026-08-21T16:47:09Z"}