{"id":{"repo_id":"mit","oai_identifier":"oai:dspace.mit.edu:1721.1/153338"},"canonical_url":"https://search.dev.ndltd.org/etd/mit/oai:dspace.mit.edu:1721.1/153338","repository":{"repo_id":"mit","name":"MIT","base_url":"https://dspace.mit.edu/oai/request"},"display":{"title":"The Effects of Pre-Training and Fine-Tuning CLIP with Domain-Specific Data","abstract":"Mercari is an online two-sided marketplace that allows users to both sell and purchase items. To create the most efficient item listing process for the sellers and bring the most relevant items to the buyers, Mercari utilizes a pre-trained model called Contrastive Language-Image Pre-training (CLIP), famed for its exceptional zero-shot performances, to support the auto-filling feature for item listing and similar items recommendation. As this model is pre-trained on a general dataset gathered from the Internet, which likely does not have the same data distribution as Mercari’s data and results in non-optimal performance, we would like to explore the possibility of pre-training or fine-tuning CLIP with Mercari’s data to improve its performance within Mercari’s data domain. We explore various training strategies to understand the effects of each and determine the most effective strategy. Our best-performing and most space-efficient model achieves a brand prediction top-1 accuracy of 89.34% with 49.89% coverage and a category prediction accuracy of 78.02% with 69.62% coverage, significantly outperforming the current zero-shot CLIP in brand prediction and marginally in category prediction. Moreover, it achieves this with an embedding size that is half of that of the original CLIP.","abstract_html":"Mercari is an online two-sided marketplace that allows users to both sell and purchase items. To create the most efficient item listing process for the sellers and bring the most relevant items to the buyers, Mercari utilizes a pre-trained model called Contrastive Language-Image Pre-training (CLIP), famed for its exceptional zero-shot performances, to support the auto-filling feature for item listing and similar items recommendation. As this model is pre-trained on a general dataset gathered from the Internet, which likely does not have the same data distribution as Mercari’s data and results in non-optimal performance, we would like to explore the possibility of pre-training or fine-tuning CLIP with Mercari’s data to improve its performance within Mercari’s data domain. We explore various training strategies to understand the effects of each and determine the most effective strategy. Our best-performing and most space-efficient model achieves a brand prediction top-1 accuracy of 89.34% with 49.89% coverage and a category prediction accuracy of 78.02% with 69.62% coverage, significantly outperforming the current zero-shot CLIP in brand prediction and marginally in category prediction. Moreover, it achieves this with an embedding size that is half of that of the original CLIP.","abstract_has_math":false,"creators":["Wang, Jialan"],"institution":"Massachusetts Institute of Technology","degree_name":"Master","degree_level":null,"degree_discipline":null,"degree_department":"Massachusetts Institute of Technology. Department of Electrical Engineering and Computer Science","school":null,"contributors":[],"advisors":["Matusik, Wojciech","Daptardar, Ajay"],"committee_chairs":[],"committee_members":[],"year":2023,"date_issued":"2023-09","date_published":"2023-09","updated_at":"2026-07-22T22:21:06Z","subjects":[],"languages":[],"rights":["In Copyright - Educational Use Permitted","Copyright retained by author(s)"],"rights_urls":["https://rightsstatements.org/page/InC-EDU/1.0/"],"identifier_entries":[]},"links":{"outbound_url":"https://hdl.handle.net/1721.1/153338","outbound_label":"Handle","outbound_source":"dc:identifier.uri"},"metadata_groups":[{"id":"people","label":"People","entries":[{"key":"dc:contributor.advisor","label":"Advisor","values":["Matusik, Wojciech","Daptardar, Ajay"]},{"key":"dc:contributor.department","label":"Department","values":["Massachusetts Institute of Technology. Department of Electrical Engineering and Computer Science"]},{"key":"dc:creator","label":"Author","values":["Wang, Jialan"]}]},{"id":"academic_context","label":"Academic Context","entries":[{"key":"dc:date.accessioned","label":"Dc Date Accessioned","values":["2024-01-16T21:52:12Z"]},{"key":"dc:date.available","label":"Dc Date Available","values":["2024-01-16T21:52:12Z"]},{"key":"dc:date.issued","label":"Date","values":["2023-09"]},{"key":"dc:publisher","label":"Institution","values":["Massachusetts Institute of Technology"]},{"key":"dc:type","label":"Dc Type","values":["Thesis"]},{"key":"thesis:degree_name","label":"Degree Name","values":["Master","Master of Engineering in Electrical Engineering and Computer Science"]}]},{"id":"language_rights","label":"Language and Rights","entries":[{"key":"dc:rights","label":"Dc Rights","values":["In Copyright - Educational Use Permitted","Copyright retained by author(s)"]},{"key":"dc:rights.uri","label":"Rights URI","values":["https://rightsstatements.org/page/InC-EDU/1.0/"]}]},{"id":"identifiers","label":"Identifiers","entries":[{"key":"dc:identifier.uri","label":"Identifier URI","values":["https://hdl.handle.net/1721.1/153338"]}]},{"id":"additional","label":"Additional Metadata","entries":[{"key":"dc:description.abstract","label":"Abstract","values":["Mercari is an online two-sided marketplace that allows users to both sell and purchase items. To create the most efficient item listing process for the sellers and bring the most relevant items to the buyers, Mercari utilizes a pre-trained model called Contrastive Language-Image Pre-training (CLIP), famed for its exceptional zero-shot performances, to support the auto-filling feature for item listing and similar items recommendation. As this model is pre-trained on a general dataset gathered from the Internet, which likely does not have the same data distribution as Mercari’s data and results in non-optimal performance, we would like to explore the possibility of pre-training or fine-tuning CLIP with Mercari’s data to improve its performance within Mercari’s data domain. We explore various training strategies to understand the effects of each and determine the most effective strategy. Our best-performing and most space-efficient model achieves a brand prediction top-1 accuracy of 89.34% with 49.89% coverage and a category prediction accuracy of 78.02% with 69.62% coverage, significantly outperforming the current zero-shot CLIP in brand prediction and marginally in category prediction. Moreover, it achieves this with an embedding size that is half of that of the original CLIP."]},{"key":"dc:description.degree","label":"Dc Description Degree","values":["M.Eng."]},{"key":"dc:title","label":"Title","values":["The Effects of Pre-Training and Fine-Tuning CLIP with Domain-Specific Data"]}]}],"canonical_facts":{"dc:contributor.advisor":["Matusik, Wojciech","Daptardar, Ajay"],"dc:contributor.department":["Massachusetts Institute of Technology. Department of Electrical Engineering and Computer Science"],"dc:creator":["Wang, Jialan"],"dc:date.accessioned":["2024-01-16T21:52:12Z"],"dc:date.available":["2024-01-16T21:52:12Z"],"dc:date.issued":["2023-09"],"dc:description.abstract":["Mercari is an online two-sided marketplace that allows users to both sell and purchase items. To create the most efficient item listing process for the sellers and bring the most relevant items to the buyers, Mercari utilizes a pre-trained model called Contrastive Language-Image Pre-training (CLIP), famed for its exceptional zero-shot performances, to support the auto-filling feature for item listing and similar items recommendation. As this model is pre-trained on a general dataset gathered from the Internet, which likely does not have the same data distribution as Mercari’s data and results in non-optimal performance, we would like to explore the possibility of pre-training or fine-tuning CLIP with Mercari’s data to improve its performance within Mercari’s data domain. We explore various training strategies to understand the effects of each and determine the most effective strategy. Our best-performing and most space-efficient model achieves a brand prediction top-1 accuracy of 89.34% with 49.89% coverage and a category prediction accuracy of 78.02% with 69.62% coverage, significantly outperforming the current zero-shot CLIP in brand prediction and marginally in category prediction. Moreover, it achieves this with an embedding size that is half of that of the original CLIP."],"dc:description.degree":["M.Eng."],"dc:identifier.uri":["https://hdl.handle.net/1721.1/153338"],"dc:publisher":["Massachusetts Institute of Technology"],"dc:rights":["In Copyright - Educational Use Permitted","Copyright retained by author(s)"],"dc:rights.uri":["https://rightsstatements.org/page/InC-EDU/1.0/"],"dc:title":["The Effects of Pre-Training and Fine-Tuning CLIP with Domain-Specific Data"],"dc:type":["Thesis"],"thesis:degree_name":["Master","Master of Engineering in Electrical Engineering and Computer Science"]},"updated_at":"2026-07-22T22:21:06Z"}