{"id":{"repo_id":"uiuc","oai_identifier":"oai:www.ideals.illinois.edu:2142/122036"},"canonical_url":"https://search.dev.ndltd.org/etd/uiuc/oai:www.ideals.illinois.edu:2142/122036","repository":{"repo_id":"uiuc","name":"University of Illinois - Urbana-Champaign","base_url":"https://www.ideals.illinois.edu/oai-pmh"},"display":{"title":"Toward building more accessible large language models: A preliminary empirical study on data scarcity in knowledge distillation and algorithm complexity in alignment","abstract":"Submission original under an indefinite embargo labeled 'Open Access'. The submission was exported from vireo on 2024-03-01 without embargo terms","abstract_html":"Submission original under an indefinite embargo labeled 'Open Access'. The submission was exported from vireo on 2024-03-01 without embargo terms","abstract_has_math":false,"creators":["Wang, Ziqi"],"institution":"University of Illinois at Urbana-Champaign","degree_name":"M.S.","degree_level":"Thesis","degree_discipline":"Computer Science","degree_department":null,"school":null,"contributors":["Ji, Heng"],"advisors":[],"committee_chairs":[],"committee_members":[],"year":2023,"date_issued":"2023-12","date_published":"2023-12","updated_at":"2026-07-22T22:25:00Z","subjects":["Large Language Models","Distillation","Data Augmentation","Alignment","Controllable Generation"],"languages":["en","eng"],"rights":["Copyright 2023 Ziqi Wang"],"rights_urls":[],"identifier_entries":[]},"links":{"outbound_url":"https://hdl.handle.net/2142/122036","outbound_label":"Handle","outbound_source":"dc:identifier"},"metadata_groups":[{"id":"people","label":"People","entries":[{"key":"dc:contributor","label":"Contributor","values":["Ji, Heng"]},{"key":"dc:creator","label":"Author","values":["Wang, Ziqi"]}]},{"id":"academic_context","label":"Academic Context","entries":[{"key":"dc:date","label":"Dc Date","values":["2023-12","2023-12-01"]},{"key":"dc:type","label":"Dc Type","values":["text"]},{"key":"thesis:degree_discipline","label":"Discipline","values":["Computer Science"]},{"key":"thesis:degree_level","label":"Degree Level","values":["Thesis"]},{"key":"thesis:degree_name","label":"Degree Name","values":["M.S."]},{"key":"thesis:institution_name","label":"Thesis Institution Name","values":["University of Illinois at Urbana-Champaign"]}]},{"id":"subjects_keywords","label":"Subjects and Keywords","entries":[{"key":"dc:subject","label":"Dc Subject","values":["Large Language Models","Distillation","Data Augmentation","Alignment","Controllable Generation"]}]},{"id":"language_rights","label":"Language and Rights","entries":[{"key":"dc:language","label":"Dc Language","values":["en","eng"]},{"key":"dc:rights","label":"Dc Rights","values":["Copyright 2023 Ziqi Wang"]}]},{"id":"identifiers","label":"Identifiers","entries":[{"key":"dc:identifier","label":"Identifier","values":["https://hdl.handle.net/2142/122036"]}]},{"id":"additional","label":"Additional Metadata","entries":[{"key":"dc:description","label":"Description","values":["Submission original under an indefinite embargo labeled 'Open Access'. The submission was exported from vireo on 2024-03-01 without embargo terms","The student, Ziqi Wang, accepted the attached license on 2023-11-29 at 20:10.","The student, Ziqi Wang, submitted this Thesis for approval on 2023-11-29 at 21:11.","This Thesis was approved for publication on 2023-12-01 at 10:11.","DSpace SAF Submission Ingestion Package generated from Vireo submission #20068 on 2024-03-01 at 13:15:14","Developing large language models (LLMs) boosts various downstream tasks such as question answering. However, for various reasons, most people have to use commercial application programming interfaces (APIs) instead of training LLMs themselves. The limited accessibility of LLMs calls for efforts in democratization. This thesis mainly explores two critical technical bottlenecks that limit LLMs' access to a broader community: data scarcity and algorithm complexity. Data scarcity makes it difficult for individuals to train their own models or distill knowledge from LLMs. Therefore, we explore suitable ways to augment text data to distill knowledge from large language models better. Besides, the complex alignment algorithm (i.e., reinforcement learning from human feedback, RLHF for short) requires lots of engineering effort, which hinders individuals from training their own models. Although there are simple substitutional algorithms, they have different drawbacks. This thesis proposes to improve controllable generation, a simple substitutional algorithm of RLHF, to achieve better alignment performance. The results of this thesis can help the community toward a more democratized LLM research environment."]},{"key":"dc:format","label":"Dc Format","values":["application/pdf"]},{"key":"dc:title","label":"Title","values":["Toward building more accessible large language models: A preliminary empirical study on data scarcity in knowledge distillation and algorithm complexity in alignment"]}]}],"canonical_facts":{"dc:contributor":["Ji, Heng"],"dc:creator":["Wang, Ziqi"],"dc:date":["2023-12","2023-12-01"],"dc:description":["Submission original under an indefinite embargo labeled 'Open Access'. The submission was exported from vireo on 2024-03-01 without embargo terms","The student, Ziqi Wang, accepted the attached license on 2023-11-29 at 20:10.","The student, Ziqi Wang, submitted this Thesis for approval on 2023-11-29 at 21:11.","This Thesis was approved for publication on 2023-12-01 at 10:11.","DSpace SAF Submission Ingestion Package generated from Vireo submission #20068 on 2024-03-01 at 13:15:14","Developing large language models (LLMs) boosts various downstream tasks such as question answering. However, for various reasons, most people have to use commercial application programming interfaces (APIs) instead of training LLMs themselves. The limited accessibility of LLMs calls for efforts in democratization. This thesis mainly explores two critical technical bottlenecks that limit LLMs' access to a broader community: data scarcity and algorithm complexity. Data scarcity makes it difficult for individuals to train their own models or distill knowledge from LLMs. Therefore, we explore suitable ways to augment text data to distill knowledge from large language models better. Besides, the complex alignment algorithm (i.e., reinforcement learning from human feedback, RLHF for short) requires lots of engineering effort, which hinders individuals from training their own models. Although there are simple substitutional algorithms, they have different drawbacks. This thesis proposes to improve controllable generation, a simple substitutional algorithm of RLHF, to achieve better alignment performance. The results of this thesis can help the community toward a more democratized LLM research environment."],"dc:format":["application/pdf"],"dc:identifier":["https://hdl.handle.net/2142/122036"],"dc:language":["en","eng"],"dc:rights":["Copyright 2023 Ziqi Wang"],"dc:subject":["Large Language Models","Distillation","Data Augmentation","Alignment","Controllable Generation"],"dc:title":["Toward building more accessible large language models: A preliminary empirical study on data scarcity in knowledge distillation and algorithm complexity in alignment"],"dc:type":["text"],"thesis:degree_discipline":["Computer Science"],"thesis:degree_level":["Thesis"],"thesis:degree_name":["M.S."],"thesis:institution_name":["University of Illinois at Urbana-Champaign"]},"updated_at":"2026-07-22T22:25:00Z"}