1 min read
SLM vs LLM: Choosing the Right Model Size Strategy
I wrote “SLM vs LLM: Choosing the Right Model Size Strategy” to share practical, production-minded guidance on this topic.
The Model Size Spectrum
Size Parameters Examples Sweet Spot
─────────────────────────────────────────────────────────────
Tiny < 1B DistilBERT Embeddings, classification
Small 1-7B Phi-3, Llama-3-8B On-device, high-volume
Medium 7-70B Llama-3-70B, Mixtral Complex tasks, balanced
Large 70B+ GPT-4, Claude Opus Reasoning, multi-modal
Frontier Unknown GPT-5, Claude 4 Research, hardest tasks
Decision Framework
def select_model_tier(task_requirements: dict) -> str:
"""Select appropriate model tier based on requirements."""
# Extract requirements
complexity = task_requirements.get("complexity", "medium")
latency_ms = task_requirements.get("max_latency_ms", 1000)
cost_per_1k = task_requirements.get("max_cost_per_1k_requests", 1.0)
accuracy_required = task_requirements.get("min_accuracy", 0.9)
volume_per_day = task_requirements.get("requests_per_day", 1000)
needs_reasoning = task_requirements.get("multi_step_reasoning", False)
context_length = task_requirements.get("context_length", 4000)
# Decision tree
if needs_reasoning and complexity == "high":
return "large" # GPT-4, Claude Opus
if latency_ms < 100:
return "small" # Must be on-device
if cost_per_1k < 0.01 and volume_per_day > 100000:
return "small" # Cost optimization
if context_length > 32000:
return "large" # Long context
if complexity == "simple":
return "tiny" if accuracy_required < 0.95 else "small"
if complexity == "medium":
return "small" if accuracy_required < 0.92 else "medium"
return "medium" # Default
# Examples
print(select_model_tier({
"complexity": "simple",
"max_latency_ms": 50,
"requests_per_day": 1000000
})) # "small"
print(select_model_tier({
"complexity": "high",
"multi_step_reasoning": True,
"min_accuracy": 0.98
})) # "large"
Cost Analysis
import pandas as pd
# Cost comparison (prices as of 2025, approximate)
model_costs = pd.DataFrame({
"Model": ["GPT-4o", "GPT-4o-mini", "Claude 3.5 Sonnet", "Phi-3-mini", "Llama-3-8B"],
"Input_per_1M": [5.00, 0.15, 3.00, 0.00, 0.00], # $ per 1M tokens
"Output_per_1M": [15.00, 0.60, 15.00, 0.00, 0.00],
"Self_hosted_per_hour": [0, 0, 0, 0.50, 1.00], # GPU costs
"Latency_avg_ms": [500, 200, 400, 50, 100],
"Quality_score": [95, 85, 93, 75, 80]
})
def calculate_monthly_cost(
model: str,
requests_per_day: int,
avg_input_tokens: int,
avg_output_tokens: int,
self_hosted: bool = False
) -> dict:
"""Calculate monthly costs for a model."""
model_data = model_costs[model_costs["Model"] == model].iloc[0]
if self_hosted:
# Self-hosted cost (assume 24/7)
monthly_cost = model_data["Self_hosted_per_hour"] * 24 * 30
else:
# API cost
daily_input_tokens = requests_per_day * avg_input_tokens
daily_output_tokens = requests_per_day * avg_output_tokens
daily_cost = (
(daily_input_tokens / 1_000_000) * model_data["Input_per_1M"] +
(daily_output_tokens / 1_000_000) * model_data["Output_per_1M"]
)
monthly_cost = daily_cost * 30
return {
"model": model,
"monthly_cost": monthly_cost,
"cost_per_request": monthly_cost / (requests_per_day * 30),
"quality_score": model_data["Quality_score"],
"avg_latency": model_data["Latency_avg_ms"]
}
# Compare scenarios
for model in ["GPT-4o", "GPT-4o-mini", "Phi-3-mini"]:
result = calculate_monthly_cost(
model=model,
requests_per_day=10000,
avg_input_tokens=500,
avg_output_tokens=200,
self_hosted=(model == "Phi-3-mini")
)
print(f"{model}: ${result['monthly_cost']:.2f}/month, Quality: {result['quality_score']}")
Hybrid Architecture
from enum import Enum
from dataclasses import dataclass
class ModelTier(Enum):
SMALL = "small"
MEDIUM = "medium"
LARGE = "large"
@dataclass
class ModelConfig:
name: str
tier: ModelTier
endpoint: str
max_tokens: int
cost_per_1k_tokens: float
class HybridModelRouter:
"""Route requests to appropriate model tier."""
def __init__(self):
self.models = {
ModelTier.SMALL: ModelConfig(
name="phi-3-mini",
tier=ModelTier.SMALL,
endpoint="http://localhost:8080",
max_tokens=4096,
cost_per_1k_tokens=0.0001
),
ModelTier.MEDIUM: ModelConfig(
name="gpt-4o-mini",
tier=ModelTier.MEDIUM,
endpoint="https://api.openai.com",
max_tokens=128000,
cost_per_1k_tokens=0.0003
),
ModelTier.LARGE: ModelConfig(
name="gpt-4o",
tier=ModelTier.LARGE,
endpoint="https://api.openai.com",
max_tokens=128000,
cost_per_1k_tokens=0.01
)
}
async def route(self, request: dict) -> ModelConfig:
"""Route request to appropriate model."""
# Simple classification
if request.get("task") in ["classification", "extraction", "embedding"]:
return self.models[ModelTier.SMALL]
# Check complexity signals
prompt_length = len(request.get("prompt", ""))
needs_reasoning = request.get("reasoning", False)
quality_requirement = request.get("quality", "standard")
if needs_reasoning or quality_requirement == "high":
return self.models[ModelTier.LARGE]
if prompt_length > 10000 or quality_requirement == "medium":
return self.models[ModelTier.MEDIUM]
return self.models[ModelTier.SMALL]
async def execute(self, request: dict) -> dict:
"""Execute request on selected model with fallback."""
model = await self.route(request)
try:
result = await self._call_model(model, request)
return result
except Exception as e:
# Fallback to larger model
if model.tier == ModelTier.SMALL:
return await self._call_model(self.models[ModelTier.MEDIUM], request)
elif model.tier == ModelTier.MEDIUM:
return await self._call_model(self.models[ModelTier.LARGE], request)
raise
Task-Specific Model Selection
# Recommended models by task
task_recommendations = {
"text_classification": {
"recommended": "phi-3-mini",
"alternative": "fine-tuned-bert",
"reasoning": "Simple task, high volume, latency sensitive"
},
"named_entity_recognition": {
"recommended": "fine-tuned-distilbert",
"alternative": "phi-3-mini",
"reasoning": "Structured output, can be fine-tuned cheaply"
},
"code_completion": {
"recommended": "phi-3-mini",
"alternative": "gpt-4o-mini",
"reasoning": "Phi excels at code, low latency for IDE"
},
"document_summarization": {
"recommended": "gpt-4o-mini",
"alternative": "claude-3-haiku",
"reasoning": "Good quality/cost balance, handles long docs"
},
"complex_analysis": {
"recommended": "gpt-4o",
"alternative": "claude-3-opus",
"reasoning": "Needs reasoning and nuance"
},
"creative_writing": {
"recommended": "claude-3-opus",
"alternative": "gpt-4o",
"reasoning": "Best creative capabilities"
},
"data_extraction_structured": {
"recommended": "gpt-4o-mini",
"alternative": "phi-3-small",
"reasoning": "Good at structured output, cost effective"
},
"real_time_chat": {
"recommended": "phi-3-mini (on-device)",
"alternative": "gpt-4o-mini",
"reasoning": "Latency is critical"
}
}
Cascade Architecture
class ModelCascade:
"""Try small model first, escalate if needed."""
def __init__(self, small_model, medium_model, large_model):
self.models = [small_model, medium_model, large_model]
self.confidence_threshold = 0.8
async def execute(self, prompt: str) -> dict:
"""Execute with cascade fallback."""
for i, model in enumerate(self.models):
result = await model.generate(prompt)
# Check if result is confident enough
confidence = await self._assess_confidence(result, prompt)
if confidence >= self.confidence_threshold:
return {
"response": result,
"model_used": model.name,
"confidence": confidence,
"cascade_level": i
}
# Return large model result regardless of confidence
return {
"response": result,
"model_used": self.models[-1].name,
"confidence": confidence,
"cascade_level": len(self.models) - 1
}
async def _assess_confidence(self, result: str, prompt: str) -> float:
"""Assess confidence in result."""
# Could use:
# - Self-consistency (generate multiple and compare)
# - Perplexity score
# - Task-specific validation
# - Small classifier model
pass
Best Practices
- Start small: Begin with smallest model that might work
- Measure quality: Establish benchmarks for your specific tasks
- Consider total cost: Include latency, infrastructure, maintenance
- Use routing: Different tasks warrant different models
- Enable fallback: Cascade to larger models when needed
- Fine-tune when possible: Custom small models often beat generic large ones
The right model isn’t always the biggest one. Match model capability to task requirements for optimal cost, latency, and quality.\n\n## Takeaways\n\nAdd a concise, personal takeaway and recommended next steps here.\n