I'll help you create a conceptual guide and code structure for building a large language model from scratch, as if it were a GitHub repository README. This is educational—actual training requires massive compute. Build a decoder-only transformer language model from scratch using PyTorch
prompt = "To be or not to be" tokens = tokenizer.encode(prompt) output = model.generate(tokens, max_new_tokens=50, temperature=0.8) print(tokenizer.decode(output)) 1. Causal Self-Attention class CausalSelfAttention(nn.Module): def __init__(self, d_model, n_heads, dropout=0.1): super().__init__() assert d_model % n_heads == 0 self.n_heads = n_heads self.head_dim = d_model // n_heads self.qkv = nn.Linear(d_model, 3 * d_model) self.proj = nn.Linear(d_model, d_model) self.dropout = nn.Dropout(dropout) # Causal mask (upper triangular) self.register_buffer("mask", torch.tril(torch.ones(1, 1, 1024, 1024)) .view(1, 1, 1024, 1024) == 0) def forward(self, x): B, T, C = x.shape qkv = self.qkv(x).chunk(3, dim=-1) q, k, v = map(lambda t: t.view(B, T, self.n_heads, self.head_dim).transpose(1, 2), qkv) att = (q @ k.transpose(-2, -1)) * (self.head_dim ** -0.5) att = att.masked_fill(self.mask[:,:,:T,:T] == 0, float('-inf')) att = F.softmax(att, dim=-1) att = self.dropout(att) y = (att @ v).transpose(1, 2).contiguous().view(B, T, C) return self.proj(y) 2. Transformer Block class TransformerBlock(nn.Module): def __init__(self, d_model, n_heads, ff_dim, dropout=0.1): super().__init__() self.ln1 = nn.LayerNorm(d_model) self.attn = CausalSelfAttention(d_model, n_heads, dropout) self.ln2 = nn.LayerNorm(d_model) self.ff = FeedForward(d_model, ff_dim, dropout) def forward(self, x): x = x + self.attn(self.ln1(x)) # Pre-LN + residual x = x + self.ff(self.ln2(x)) return x 3. BPE Tokenizer (simplified) class BPETokenizer: def train(self, text, vocab_size=5000): # Start with byte-level tokens self.vocab = idx: bytes([idx]) for idx in range(256) self.merges = {} # Split into words words = [list(word.encode('utf-8')) + [0] for word in text.split()] # Iteratively merge most frequent pairs for i in range(256, vocab_size): pairs = self._get_stats(words) if not pairs: break best_pair = max(pairs, key=pairs.get) self.merges[best_pair] = i words = self._merge_pair(words, best_pair, i) Training Configuration # configs/medium.yaml model: d_model: 768 n_heads: 12 n_layers: 12 ff_dim: 3072 vocab_size: 50257 max_seq_len: 1024 dropout: 0.1 training: batch_size: 32 learning_rate: 3e-4 total_steps: 50000 warmup_steps: 500 weight_decay: 0.1 grad_clip: 1.0 build a large language model from scratch github