{"id":{"repo_id":"nus","oai_identifier":"oai:scholarbank.nus.edu.sg:10635/166275"},"canonical_url":"https://search.dev.ndltd.org/etd/nus/oai:scholarbank.nus.edu.sg:10635/166275","repository":{"repo_id":"nus","name":"National University of Singapore","base_url":"https://scholarbank.nus.edu.sg/oai/request"},"display":{"title":"REGULARIZATION ON MACHINE LEARNING","abstract":"Deep neural networks have become a powerful tool for machine learning problems. However, overfitting frequently occurs. To achieve better generalization, many regularization methods were proposed to reduce overfitting. In this thesis, we propose a simple-yet-effective regularization method called Drop-Activation. At the training phase, we drop nonlinear activation functions randomly and set them to be identity functions. At the testing phase, a deterministic network with a new activation function is used and the new activation function is designed to average effect of the randomness of discarding activations. We theoretically deduce the implicit regularization terms of Drop-Activation and the effect of Drop-Activation can be considered as implicit parameter reduction. Also, our theoretical analysis verifies its capability to be used together with Batch Normalization (Ioffe and Szegedy 2015). We perform Drop-Activation on the benchmark datasets and show that the performance of popular networks can be improved generally by Drop-Activation.","abstract_html":"Deep neural networks have become a powerful tool for machine learning problems. However, overfitting frequently occurs. To achieve better generalization, many regularization methods were proposed to reduce overfitting. In this thesis, we propose a simple-yet-effective regularization method called Drop-Activation. At the training phase, we drop nonlinear activation functions randomly and set them to be identity functions. At the testing phase, a deterministic network with a new activation function is used and the new activation function is designed to average effect of the randomness of discarding activations. We theoretically deduce the implicit regularization terms of Drop-Activation and the effect of Drop-Activation can be considered as implicit parameter reduction. Also, our theoretical analysis verifies its capability to be used together with Batch Normalization (Ioffe and Szegedy 2015). We perform Drop-Activation on the benchmark datasets and show that the performance of popular networks can be improved generally by Drop-Activation.","abstract_has_math":false,"creators":["LIANG SENWEI"],"institution":null,"degree_name":null,"degree_level":null,"degree_discipline":null,"degree_department":null,"school":null,"contributors":[],"advisors":[],"committee_chairs":[],"committee_members":[],"year":2019,"date_issued":"2019-12-18","date_published":"2019-12-18","updated_at":"2026-07-24T03:30:34Z","subjects":["deep learning, regularization, generalization, overfitting, Drop-Activation"],"languages":[],"rights":[],"rights_urls":[],"identifier_entries":[]},"links":{"outbound_url":null,"outbound_label":null,"outbound_source":null},"metadata_groups":[{"id":"people","label":"People","entries":[{"key":"dc:creator","label":"Author","values":["LIANG SENWEI"]}]},{"id":"academic_context","label":"Academic Context","entries":[{"key":"dc:date.issued","label":"Date","values":["2019-12-18"]},{"key":"dc:relation.isreferencedby","label":"Dc Relation Isreferencedby","values":["https://scholarbank.nus.edu.sg/handle/10635/166275"]},{"key":"dc:type","label":"Dc Type","values":["Thesis"]}]},{"id":"subjects_keywords","label":"Subjects and Keywords","entries":[{"key":"dc:subject","label":"Dc Subject","values":["deep learning, regularization, generalization, overfitting, Drop-Activation"]}]},{"id":"identifiers","label":"Identifiers","entries":[{"key":"dc:identifier.uri","label":"Identifier URI","values":["https://scholarbank.nus.edu.sg/bitstreams/3f2441a9-8bec-4f85-baf1-45e6c4294d27/download"]}]},{"id":"additional","label":"Additional Metadata","entries":[{"key":"dc:description.abstract","label":"Abstract","values":["Deep neural networks have become a powerful tool for machine learning problems. However, overfitting frequently occurs. To achieve better generalization, many regularization methods were proposed to reduce overfitting. In this thesis, we propose a simple-yet-effective regularization method called Drop-Activation. At the training phase, we drop nonlinear activation functions randomly and set them to be identity functions. At the testing phase, a deterministic network with a new activation function is used and the new activation function is designed to average effect of the randomness of discarding activations. We theoretically deduce the implicit regularization terms of Drop-Activation and the effect of Drop-Activation can be considered as implicit parameter reduction. Also, our theoretical analysis verifies its capability to be used together with Batch Normalization (Ioffe and Szegedy 2015). We perform Drop-Activation on the benchmark datasets and show that the performance of popular networks can be improved generally by Drop-Activation."]},{"key":"dc:format.checksum.md5","label":"Dc Format Checksum Md5","values":["97978810ddc46456f40627f6c35fea47","639d7e8f3157c6b35f1279cc9dff717e"]},{"key":"dc:title","label":"Title","values":["REGULARIZATION ON MACHINE LEARNING"]}]}],"canonical_facts":{"dc:creator":["LIANG SENWEI"],"dc:date.issued":["2019-12-18"],"dc:description.abstract":["Deep neural networks have become a powerful tool for machine learning problems. However, overfitting frequently occurs. To achieve better generalization, many regularization methods were proposed to reduce overfitting. In this thesis, we propose a simple-yet-effective regularization method called Drop-Activation. At the training phase, we drop nonlinear activation functions randomly and set them to be identity functions. At the testing phase, a deterministic network with a new activation function is used and the new activation function is designed to average effect of the randomness of discarding activations. We theoretically deduce the implicit regularization terms of Drop-Activation and the effect of Drop-Activation can be considered as implicit parameter reduction. Also, our theoretical analysis verifies its capability to be used together with Batch Normalization (Ioffe and Szegedy 2015). We perform Drop-Activation on the benchmark datasets and show that the performance of popular networks can be improved generally by Drop-Activation."],"dc:format.checksum.md5":["97978810ddc46456f40627f6c35fea47","639d7e8f3157c6b35f1279cc9dff717e"],"dc:identifier.uri":["https://scholarbank.nus.edu.sg/bitstreams/3f2441a9-8bec-4f85-baf1-45e6c4294d27/download"],"dc:relation.isreferencedby":["https://scholarbank.nus.edu.sg/handle/10635/166275"],"dc:subject":["deep learning, regularization, generalization, overfitting, Drop-Activation"],"dc:title":["REGULARIZATION ON MACHINE LEARNING"],"dc:type":["Thesis"]},"updated_at":"2026-07-24T03:30:34Z"}