{"id":{"repo_id":"vt","oai_identifier":"oai:vtechworks.lib.vt.edu:10919/115712"},"canonical_url":"https://search.dev.ndltd.org/etd/vt/oai:vtechworks.lib.vt.edu:10919/115712","repository":{"repo_id":"vt","name":"Virginia Tech","base_url":"https://vtechworks.lib.vt.edu/oai/request"},"display":{"title":"Study of Pretraining Bias and Frequencies","abstract":"Usage of language models in an in-context learning environment has been adapted for a wide range of tasks. Recent works have showcased the impact of pretraining data on the in-context performance of language models. In this work, we experiment with numbers having high and low frequencies in the pretraining data to understand the impact of term frequencies on the model's performance. We also experiment with random and adversarial demonstrations to understand the pretraining bias present in the model. Through these experiments, we showcase the importance of pretraining frequencies of the numbers present in the demonstrations and explain how highly frequent terms can be used in the demonstrations to achieve better task performance. Moreover, we also show the impact of pretraining bias on the model's performance and explain how the model overcomes this bias with more demonstrations.","abstract_html":"Usage of language models in an in-context learning environment has been adapted for a wide range of tasks. Recent works have showcased the impact of pretraining data on the in-context performance of language models. In this work, we experiment with numbers having high and low frequencies in the pretraining data to understand the impact of term frequencies on the model's performance. We also experiment with random and adversarial demonstrations to understand the pretraining bias present in the model. Through these experiments, we showcase the importance of pretraining frequencies of the numbers present in the demonstrations and explain how highly frequent terms can be used in the demonstrations to achieve better task performance. Moreover, we also show the impact of pretraining bias on the model's performance and explain how the model overcomes this bias with more demonstrations.","abstract_has_math":false,"creators":["Taware, Rutuja Murlidhar"],"institution":"Virginia Tech","degree_name":"Master of Science","degree_level":"masters","degree_discipline":"Computer Science and Applications","degree_department":"Computer Science and Applications","school":null,"contributors":[],"advisors":[],"committee_chairs":["Ramakrishnan, Narendran"],"committee_members":["Lourentzou, Ismini","Lu, Chang Tien"],"year":2023,"date_issued":"2023-07-10","date_published":"2023-07-10","updated_at":"2026-07-22T22:19:33Z","subjects":["In-Context Learning","Pretraining","Frequency","Bias","Language Model"],"languages":["en"],"rights":["In Copyright"],"rights_urls":["http://rightsstatements.org/vocab/InC/1.0/"],"identifier_entries":[{"key":"dc:identifier.other","label":"Dc Identifier Other","values":["vt_gsexam:37728"],"render_values":[{"text":"vt_gsexam:37728","href":null,"code":true}]}]},"links":{"outbound_url":"http://hdl.handle.net/10919/115712","outbound_label":"Handle","outbound_source":"dc:identifier.uri"},"metadata_groups":[{"id":"people","label":"People","entries":[{"key":"dc:contributor.committeechair","label":"Committee Chair","values":["Ramakrishnan, Narendran"]},{"key":"dc:contributor.committeemember","label":"Committee Member","values":["Lourentzou, Ismini","Lu, Chang Tien"]},{"key":"dc:contributor.department","label":"Department","values":["Computer Science and Applications"]},{"key":"dc:creator","label":"Author","values":["Taware, Rutuja Murlidhar"]}]},{"id":"academic_context","label":"Academic Context","entries":[{"key":"dc:date.accessioned","label":"Dc Date Accessioned","values":["2023-07-11T08:00:33Z"]},{"key":"dc:date.available","label":"Dc Date Available","values":["2023-07-11T08:00:33Z"]},{"key":"dc:date.issued","label":"Date","values":["2023-07-10"]},{"key":"dc:publisher","label":"Institution","values":["Virginia Tech"]},{"key":"dc:type","label":"Dc Type","values":["Thesis"]},{"key":"thesis:degree_discipline","label":"Discipline","values":["Computer Science and Applications"]},{"key":"thesis:degree_level","label":"Degree Level","values":["masters"]},{"key":"thesis:degree_name","label":"Degree Name","values":["Master of Science"]},{"key":"thesis:institution_name","label":"Thesis Institution Name","values":["Virginia Polytechnic Institute and State University"]}]},{"id":"subjects_keywords","label":"Subjects and Keywords","entries":[{"key":"dc:subject","label":"Dc Subject","values":["In-Context Learning","Pretraining","Frequency","Bias","Language Model"]}]},{"id":"language_rights","label":"Language and Rights","entries":[{"key":"dc:language.iso","label":"Language (ISO)","values":["en"]},{"key":"dc:rights","label":"Dc Rights","values":["In Copyright"]},{"key":"dc:rights.uri","label":"Rights URI","values":["http://rightsstatements.org/vocab/InC/1.0/"]}]},{"id":"identifiers","label":"Identifiers","entries":[{"key":"dc:identifier.other","label":"Dc Identifier Other","values":["vt_gsexam:37728"]},{"key":"dc:identifier.uri","label":"Identifier URI","values":["http://hdl.handle.net/10919/115712"]}]},{"id":"additional","label":"Additional Metadata","entries":[{"key":"dc:description.abstract","label":"Abstract","values":["Usage of language models in an in-context learning environment has been adapted for a wide range of tasks. Recent works have showcased the impact of pretraining data on the in-context performance of language models. In this work, we experiment with numbers having high and low frequencies in the pretraining data to understand the impact of term frequencies on the model's performance. We also experiment with random and adversarial demonstrations to understand the pretraining bias present in the model. Through these experiments, we showcase the importance of pretraining frequencies of the numbers present in the demonstrations and explain how highly frequent terms can be used in the demonstrations to achieve better task performance. Moreover, we also show the impact of pretraining bias on the model's performance and explain how the model overcomes this bias with more demonstrations."]},{"key":"dc:description.abstractgeneral","label":"General Abstract","values":["Recent works focus on understanding and improving the arithmetic capabilities of the state-of-the-art (SOTA) systems in the domain of Natural Language Processing (NLP). This work focuses on designing and performing novel experiments to analyze the impact of training data on the performance of such systems. Through these experiments, this work showcases interesting properties of the SOTA systems which will promote future research to understand them better as well as help in creating better downstream applications."]},{"key":"dc:description.degree","label":"Dc Description Degree","values":["Master of Science"]},{"key":"dc:format.medium","label":"Dc Format Medium","values":["ETD"]},{"key":"dc:title","label":"Title","values":["Study of Pretraining Bias and Frequencies"]}]}],"canonical_facts":{"dc:contributor.committeechair":["Ramakrishnan, Narendran"],"dc:contributor.committeemember":["Lourentzou, Ismini","Lu, Chang Tien"],"dc:contributor.department":["Computer Science and Applications"],"dc:creator":["Taware, Rutuja Murlidhar"],"dc:date.accessioned":["2023-07-11T08:00:33Z"],"dc:date.available":["2023-07-11T08:00:33Z"],"dc:date.issued":["2023-07-10"],"dc:description.abstract":["Usage of language models in an in-context learning environment has been adapted for a wide range of tasks. Recent works have showcased the impact of pretraining data on the in-context performance of language models. In this work, we experiment with numbers having high and low frequencies in the pretraining data to understand the impact of term frequencies on the model's performance. We also experiment with random and adversarial demonstrations to understand the pretraining bias present in the model. Through these experiments, we showcase the importance of pretraining frequencies of the numbers present in the demonstrations and explain how highly frequent terms can be used in the demonstrations to achieve better task performance. Moreover, we also show the impact of pretraining bias on the model's performance and explain how the model overcomes this bias with more demonstrations."],"dc:description.abstractgeneral":["Recent works focus on understanding and improving the arithmetic capabilities of the state-of-the-art (SOTA) systems in the domain of Natural Language Processing (NLP). This work focuses on designing and performing novel experiments to analyze the impact of training data on the performance of such systems. Through these experiments, this work showcases interesting properties of the SOTA systems which will promote future research to understand them better as well as help in creating better downstream applications."],"dc:description.degree":["Master of Science"],"dc:format.medium":["ETD"],"dc:identifier.other":["vt_gsexam:37728"],"dc:identifier.uri":["http://hdl.handle.net/10919/115712"],"dc:language.iso":["en"],"dc:publisher":["Virginia Tech"],"dc:rights":["In Copyright"],"dc:rights.uri":["http://rightsstatements.org/vocab/InC/1.0/"],"dc:subject":["In-Context Learning","Pretraining","Frequency","Bias","Language Model"],"dc:title":["Study of Pretraining Bias and Frequencies"],"dc:type":["Thesis"],"thesis:degree_discipline":["Computer Science and Applications"],"thesis:degree_level":["masters"],"thesis:degree_name":["Master of Science"],"thesis:institution_name":["Virginia Polytechnic Institute and State University"]},"updated_at":"2026-07-22T22:19:33Z"}