Finetunning an open source model on your own data: Part 1

def find_files(directory, extension="*.py"):
    "Find all files of a given `extension` in a directory and return their content and path"
    md_files = []
    for file in Path(directory).rglob(extension):
        with open(file, 'r', encoding='utf-8') as md_file:
            content = md_file.read()
	# we store the path and the file content 😎
        md_files.append((file.relative_to(directory), content))
    return md_files


vllm_path = Path("vllm/"). # I cloned the vllm repo (git clone https://github.com/vllm-project/vllm.git)
py_files = find_files(vllm_path). # we get 96 python files

stuff = """
<<Begin file>>
Path:{path}
---------
Content:
{content}
<<End File>>"""

import json
from tqdm.auto import tqdm

with open('vllm_python.jsonl', 'w') as json_file:
    for path, content in tqdm(py_files):
        data = stuff.format(path=path, content=content)
        json.dump({"text":data}, json_file)
        json_file.write('\n')

from transformers import AutoTokenizer
OS_MODEL = "codellama/CodeLlama-7b-Python-hf". # All llama models share the same tokenizer with 32k vocab

# we load the tokenizer
tokenizer = AutoTokenizer.from_pretrained(OS_MODEL)

# we can call it like so
tokenized_sentence = tokenizer.encode("def hello_world():\n\tprint('Hello World!')")

# len(tokenized_sentence): 14 tokens
# tokenized_sentend: [1, 822, 22172, 29918, 11526, 7295, 13, 12, 2158, 877, 10994, 2787, 29991, 1495]

import json
def read_jsonl(fname):
    "Read a .jsonl file and return a list of dicts"
    with open(fname, 'r') as json_file:
        return [json.loads(line) for line in json_file]'

data = read_jsonl("vllm_python.jsonl")

# stacj everything together on a single bit string!
raw_corpus = "\n".join([d['text'] for d in data])

# len(raw_corpus): 604130

tokenized_data = tokenizer.encode(raw_corpus)
tokens = len(tokenized_data)
print(f"VLLM .py files total tokens: {tokens/1_000_000}M")

# VLLM .py files total tokens: 0.184048M. <-- this is not a lot on LLM standards... 🤔

import wandb

with wandb.init(project="vllm_llm"):
    at = wandb.Artifact(name="vllm_python", 
                        description="The .py files from the vllm library",
                        type="dataset",
                        metadata={
                            "url": "https://github.com/vllm-project/vllm.git",
                            "commit":last_commit,             # let's add some info about the codebase, like the commit
                            "remote": vllm_repo.remote().url, # the URL for the codebase
                            "tokens": tokens})                # and the total tokens we have
    at.add_file("vllm_python.jsonl")
    
    wandb.log_artifact(at)

# ===== Before =====
# layer = nn.Linear(in_features, out_features)

# ===== After ======
# Add a pair of low-rank adaptation matrices with rank r=16
layer = lora.Linear(in_features, out_features, r=16)

LlamaForCausalLM(
  (model): LlamaModel(
    (embed_tokens): Embedding(32016, 4096)
    (layers): ModuleList(
      (0-31): 32 x LlamaDecoderLayer(
        (self_attn): LlamaAttention(
          (q_proj): Linear(in_features=4096, out_features=4096, bias=False)
          (k_proj): Linear(in_features=4096, out_features=4096, bias=False)
          (v_proj): Linear(in_features=4096, out_features=4096, bias=False)
          (o_proj): Linear(in_features=4096, out_features=4096, bias=False)
          (rotary_emb): LlamaRotaryEmbedding()
        )
        (mlp): LlamaMLP(
          (gate_proj): Linear(in_features=4096, out_features=11008, bias=False)
          (up_proj): Linear(in_features=4096, out_features=11008, bias=False)
          (down_proj): Linear(in_features=11008, out_features=4096, bias=False)
          (act_fn): SiLUActivation()
        )
        (input_layernorm): LlamaRMSNorm()
        (post_attention_layernorm): LlamaRMSNorm()
      )
    )
    (norm): LlamaRMSNorm()
  )
  (lm_head): Linear(in_features=4096, out_features=32016, bias=False)
)