Llama 4: Meta's Shift to Multimodal MoE

Llama 4 claims a 10 million token context window — 80x larger than Llama 3’s 128K. If true, this is not incremental progress; it is a complete reinvention of attention mechanisms, KV cache management, and position encoding. Standard RoPE breaks down past 1M tokens, FlashAttention still costs $O(n^2)$ FLOPs even with IO optimization, and storing 10M tokens of KV cache at FP16 requires 320 GB per layer. Meta either invented subquadratic attention that preserves quality, or the 10M claim applies only to retrieval-augmented contexts where most tokens are not actively attended.

Model Specifications

class Llama4Config:
    """Llama 4 model family configuration."""

    scout = {
        'name': 'Llama 4 Scout',
        'total_params': 109e9,
        'active_params': 17e9,
        'num_layers': 48,
        'hidden_size': 5120,
        'num_attention_heads': 40,
        'num_kv_heads': 8,      # GQA: 5 query heads per KV head
        'head_dim': 128,
        'num_experts': 16,
        'top_k': 1,             # Top-1 routing (unusual choice)
        'expert_intermediate': 8192,
        'context_length': 10_000_000,  # 10M tokens claimed
        'vocab_size': 202_000,
        'modalities': ['text', 'image'],
        'vision_encoder': 'MetaCLIP-based',
    }

    maverick = {
        'name': 'Llama 4 Maverick',
        'total_params': 400e9,
        'active_params': 17e9,   # Same active params as Scout
        'num_layers': 48,
        'hidden_size': 5120,
        'num_attention_heads': 40,
        'num_kv_heads': 8,
        'head_dim': 128,
        'num_experts': 128,      # 8x more experts than Scout
        'top_k': 1,
        'expert_intermediate': 4096,  # Smaller per-expert FFN
        'context_length': 1_000_000,
        'vocab_size': 202_000,
        'modalities': ['text', 'image'],
    }

Top-1 Routing Analysis

class Top1RoutingAnalysis:
    """
    Llama 4's top-1 routing is a deliberate serving cost optimization.
    """

    def compute_efficiency(self):
        configs = {
            'Llama 4 Scout (top-1, 16E)': {
                'experts': 16, 'top_k': 1,
                'expert_dim': 8192, 'hidden': 5120,
                'active_expert_flops': 1 * 3 * 5120 * 8192 * 2,
            },
            'Mixtral (top-2, 8E)': {
                'experts': 8, 'top_k': 2,
                'expert_dim': 14336, 'hidden': 4096,
                'active_expert_flops': 2 * 3 * 4096 * 14336 * 2,
            },
            'DeepSeek V3 (top-8, 256E)': {
                'experts': 256, 'top_k': 8,
                'expert_dim': 2048, 'hidden': 7168,
                'active_expert_flops': 8 * 3 * 7168 * 2048 * 2,
            },
        }

        for name, cfg in configs.items():
            from math import comb
            combos = comb(cfg['experts'], cfg['top_k'])
            active_gflops = cfg['active_expert_flops'] / 1e9

            print(f"{name}:")
            print(f"  Active expert FLOPs: {active_gflops:.1f} GFLOPS/token/layer")
            print(f"  Routing combinations: {combos}")
            print(f"  Active fraction: {cfg['top_k']/cfg['experts']:.1%}")

    def top1_vs_topk_quality(self):
        """
        Top-1 routing concerns:
        1. Less routing expressiveness (16 choices vs C(16,2)=120 for top-2)
        2. Each token gets a single expert's perspective
        3. Load balancing is harder (each token must go to exactly one expert)

        Mitigations in Llama 4:
        1. Shared expert: a dense FFN processed by ALL tokens, providing
           a common baseline that experts specialize on top of
        2. More layers (48): each token sees 48 different expert selections
        3. Larger expert FFN (8192 intermediate): each expert has more capacity
        """
        pass

Shared Expert Architecture

class Llama4MoELayer(nn.Module):
    """
    Llama 4 MoE layer with shared expert.
    The shared expert processes ALL tokens, providing a common
    baseline representation. The routed expert adds specialization.
    """

    def __init__(self, config):
        super().__init__()
        self.num_experts = config['num_experts']

        # Router
        self.router = nn.Linear(config['hidden_size'], self.num_experts, bias=False)

        # Shared expert: processed by every token
        self.shared_expert = SwiGLUExpert(
            config['hidden_size'],
            config['expert_intermediate']
        )

        # Routed experts: one selected per token
        self.routed_experts = nn.ModuleList([
            SwiGLUExpert(config['hidden_size'], config['expert_intermediate'])
            for _ in range(self.num_experts)
        ])

    def forward(self, x):
        batch_size, hidden_dim = x.shape

        # Shared expert: all tokens
        shared_output = self.shared_expert(x)

        # Router: select one expert per token
        router_logits = self.router(x)
        routing_weights = torch.softmax(router_logits, dim=-1)
        top1_weight, top1_index = routing_weights.max(dim=-1)

        # Routed expert computation
        routed_output = torch.zeros_like(x)
        for e in range(self.num_experts):
            mask = (top1_index == e)
            if mask.any():
                expert_out = self.routed_experts[e](x[mask])
                routed_output[mask] = expert_out

        # Combine: shared + weighted routed
        output = shared_output + top1_weight.unsqueeze(-1) * routed_output

        return output

Multimodal Architecture

class Llama4VisionEncoder:
    """
    Llama 4's vision encoder processes images into token embeddings
    that are interleaved with text tokens in the main transformer.
    """

    def __init__(self):
        # MetaCLIP-based vision encoder
        self.patch_size = 14       # 14x14 pixel patches
        self.image_size = 560      # 560x560 input images
        self.num_patches = (560 // 14) ** 2  # 1,600 patches
        self.vision_hidden = 1408  # Vision encoder hidden dim

        # Projection from vision to language space
        self.vision_projection = nn.Linear(1408, 5120)  # -> Llama hidden dim

    def encode_image(self, image):
        """
        image: [batch, 3, 560, 560]
        Returns: [batch, 1600, 5120] — image tokens in language space
        """
        # Patch embedding + vision transformer
        patches = self.patch_embed(image)    # [batch, 1600, 1408]
        vision_features = self.vision_transformer(patches)

        # Project to language model dimension
        image_tokens = self.vision_projection(vision_features)  # [batch, 1600, 5120]

        return image_tokens

    def interleave_with_text(self, text_tokens, image_tokens, image_positions):
        """
        Insert image tokens at specified positions in the text sequence.
        """
        # text_tokens: [batch, text_len, 5120]
        # image_tokens: [batch, 1600, 5120]
        # image_positions: where to insert the image
        combined = []
        for pos, token in enumerate(text_tokens):
            if pos in image_positions:
                combined.extend(image_tokens)
            combined.append(token)
        return combined

Serving Performance

Quality Results

Training Infrastructure for Llama 4

def llama4_training_details():
    """
    Meta trained Llama 4 on their Grand Teton clusters.
    """
    training_config = {
        'gpu_cluster': '24,576 H100 GPUs',
        'interconnect': 'NVLink + InfiniBand NDR 400G',
        'training_precision': 'BF16 with FP8 expert computation',
        'sequence_length': 32768,  # Training context
        'global_batch_tokens': 16_000_000,

        # Parallelism for MoE
        'data_parallel': 256,
        'tensor_parallel': 8,     # Within node
        'expert_parallel': 16,    # Experts distributed across GPUs
        'pipeline_parallel': 1,   # No PP for MoE (complex scheduling)

        # Training data
        'pretraining_tokens': '~30T estimated',
        'multimodal_tokens': '~5T (image-text pairs)',
        'post_training': 'RLHF + DPO',
    }
    return training_config

Competitive Positioning

def llama4_competitive_analysis():
    """
    Where Llama 4 sits in the frontier model landscape.
    """
    positioning = {
        'vs_deepseek_v3': {
            'advantage': 'Open-weight, multimodal, simpler routing (top-1)',
            'disadvantage': 'Lower quality on text benchmarks (82 vs 84 MMLU)',
            'note': 'DeepSeek uses top-8 routing for better quality',
        },
        'vs_gpt4o': {
            'advantage': 'Open-weight, free to deploy, customizable',
            'disadvantage': 'Lower quality across most benchmarks',
            'note': 'Cost advantage for self-hosted deployment',
        },
        'vs_mixtral': {
            'advantage': 'Native multimodal, larger expert pool, longer context',
            'disadvantage': 'More memory for Maverick variant',
            'note': 'Scout is the natural Mixtral successor',
        },
    }
    return positioning

Llama 4 demonstrates that Meta’s MoE transition was the right engineering decision. Scout delivers 79.6 MMLU with 17B active parameters on a single GPU, while the dense predecessor Llama 3.1 405B required 4 GPUs for 87.3 MMLU. Maverick pushes to 82.1 MMLU while maintaining the same 17B active footprint by scaling total parameters to 400B across 128 experts. The addition of native vision capabilities positions Llama 4 against multimodal competitors like GPT-4o. The key architectural choice — top-1 routing with shared experts — prioritizes serving simplicity over routing expressiveness, a pragmatic decision for Meta’s goal of widespread open-source deployment.