REAL-WORLD CASE STUDIES

For each system, we will examine:
1. Requirements and constraints
2. Architecture decisions
3. Production challenges
4. How principles apply

- 500 million daily active users
- 100 million new posts per day
- 10 billion feed reads per day
- Average user follows 500 accounts

def create_post(user_id, content):
    # Store post
    post_id = post_store.create(user_id, content)

    # Get all followers
    followers = graph.get_followers(user_id)

    # Push to each follower's feed cache
    for follower_id in followers:
        feed_cache.prepend(follower_id, post_id)

Post creation time: 100M cache writes
Even at 100K writes/sec = 1000 seconds
Unacceptable latency

def get_feed(user_id):
    # Get who user follows
    following = graph.get_following(user_id)

    # Get recent posts from each
    posts = []
    for followed_id in following:
        posts.extend(post_store.get_recent(followed_id, limit=10))

    # Sort and return
    return sorted(posts, key=lambda p: p.created_at, reverse=True)[:50]

Feed load: 1000 database queries
Even with batching = slow

- Regular users (< 10K followers): Push
- Celebrities (> 10K followers): Pull at read time

Feed = Cached pushed posts + Pulled celebrity posts

CELEBRITY_THRESHOLD = 10000

def create_post(user_id, content):
    post_id = post_store.create(user_id, content)

    follower_count = graph.get_follower_count(user_id)

    if follower_count < CELEBRITY_THRESHOLD:
        # Push to followers
        fanout_queue.enqueue("fanout_post", {
            "post_id": post_id,
            "user_id": user_id
        })
    else:
        # Celebrity: do not push, will be pulled
        celebrity_posts_cache.add(user_id, post_id)

def get_feed(user_id):
    # Get cached feed (pushed posts)
    feed = feed_cache.get(user_id)

    # Get celebrities user follows
    celebrities = graph.get_following_celebrities(user_id)

    # Pull celebrity posts
    for celeb_id in celebrities:
        celeb_posts = celebrity_posts_cache.get_recent(celeb_id)
        feed.extend(celeb_posts)

    # Sort and return
    return sorted(feed, key=lambda p: p.created_at, reverse=True)[:50]

def rank_feed(user_id, posts):
    user_prefs = get_user_preferences(user_id)

    scored_posts = []
    for post in posts:
        score = 0

        # Recency (decay over time)
        age_hours = (now() - post.created_at).hours
        score += 100 * math.exp(-age_hours / 24)

        # Engagement
        score += post.likes * 0.1 + post.comments * 0.5

        # Relationship strength
        interaction_score = get_interaction_score(user_id, post.author_id)
        score += interaction_score * 50

        scored_posts.append((score, post))

    return [p for _, p in sorted(scored_posts, reverse=True)]

def on_new_post(post):
    # Invalidate feed caches for followers
    followers = graph.get_followers(post.author_id, limit=10000)

    for follower_batch in batch(followers, 1000):
        # Async cache invalidation
        cache_invalidation_queue.enqueue(
            "invalidate_feeds",
            {"user_ids": follower_batch}
        )

def invalidate_feeds(user_ids):
    for user_id in user_ids:
        # Delete cached feed, will be rebuilt on next request
        feed_cache.delete(user_id)

def warm_active_user_feeds():
    """Run periodically for most active users."""
    active_users = get_most_active_users(limit=100000)

    for user_id in active_users:
        if not feed_cache.exists(user_id):
            feed = compute_feed(user_id)
            feed_cache.set(user_id, feed, ttl=3600)

- PCI DSS compliance required
- Multiple payment providers (Stripe, PayPal, etc.)
- Multi-currency support
- Fraud detection integration
- Regulatory requirements vary by country

class PaymentOrchestrator:
    def process_payment(self, request):
        idempotency_key = request.idempotency_key

        # Check if already processed
        existing = self.idempotency_store.get(idempotency_key)
        if existing:
            return existing.response

        # Lock to prevent concurrent processing
        with self.idempotency_store.lock(idempotency_key, timeout=30):
            # Double-check after acquiring lock
            existing = self.idempotency_store.get(idempotency_key)
            if existing:
                return existing.response

            # Process payment
            try:
                result = self._execute_payment(request)

                # Store result
                self.idempotency_store.set(
                    idempotency_key,
                    {"status": "success", "response": result},
                    ttl=timedelta(days=7)
                )

                return result
            except Exception as e:
                self.idempotency_store.set(
                    idempotency_key,
                    {"status": "failed", "error": str(e)},
                    ttl=timedelta(hours=1)
                )
                raise

class Payment:
    VALID_TRANSITIONS = {
        "created": ["processing"],
        "processing": ["authorized", "failed"],
        "authorized": ["captured", "voided"],
        "captured": ["settled", "refunded"],
        "settled": ["refunded"],
    }

    def transition_to(self, new_state):
        if new_state not in self.VALID_TRANSITIONS.get(self.state, []):
            raise InvalidStateTransition(
                f"Cannot transition from{self.state} to{new_state}"
            )

        old_state = self.state
        self.state = new_state
        self.state_history.append({
            "from": old_state,
            "to": new_state,
            "timestamp": now(),
        })

        self.ledger.record_transition(self.id, old_state, new_state)

class PaymentGateway:
    PROVIDERS = ["stripe", "adyen", "braintree"]

    def charge(self, request):
        errors = []

        for provider in self.PROVIDERS:
            if not self.is_provider_healthy(provider):
                continue

            try:
                return self.providers[provider].charge(request)
            except ProviderUnavailableError as e:
                errors.append((provider, e))
                self.mark_provider_unhealthy(provider)
                continue
            except CardDeclinedError:
                # Do not retry declined cards with other providers
                raise

        raise AllProvidersFailedError(errors)

class Reconciliation:
    def daily_reconciliation(self, date):
        """Compare internal ledger with provider records."""

        discrepancies = []

        for provider in PROVIDERS:
            # Get our records
            our_transactions = self.ledger.get_transactions(
                provider=provider,
                date=date
            )

            # Get provider records
            their_transactions = provider.get_settlement_report(date)

            # Compare
            for our_tx in our_transactions:
                their_tx = their_transactions.get(our_tx.provider_id)

                if not their_tx:
                    discrepancies.append({
                        "type": "missing_at_provider",
                        "transaction": our_tx
                    })
                elif our_tx.amount != their_tx.amount:
                    discrepancies.append({
                        "type": "amount_mismatch",
                        "our_amount": our_tx.amount,
                        "their_amount": their_tx.amount
                    })

        if discrepancies:
            alert_finance_team(discrepancies)

        return discrepancies

- 50 million concurrent connections
- 10 billion messages per day
- 1 million messages per second at peak

class ChatService:
    def send_message(self, sender_id, conversation_id, content):
        # Generate message ID
        message_id = generate_snowflake_id()  # Ordered, unique

        message = {
            "id": message_id,
            "sender_id": sender_id,
            "conversation_id": conversation_id,
            "content": content,
            "timestamp": now(),
        }

        # Persist first (durability)
        self.message_store.save(message)

        # Get conversation members
        members = self.get_conversation_members(conversation_id)

        # Deliver to each member
        for member_id in members:
            self.deliver_to_user(member_id, message)

    def deliver_to_user(self, user_id, message):
        # Find which server user is connected to
        server_id = self.registry.get(f"user:{user_id}:server")

        if server_id:
            # User is online, push message
            self.push_to_server(server_id, user_id, message)
        else:
            # User is offline, queue for later
            self.offline_queue.enqueue(user_id, message)

Messages sent: A, B, C
User receives: A, C, B (out of order!)

Causes:
- Different network paths
- Load balancer distribution
- Race conditions

class ConversationSequencer:
    def get_next_sequence(self, conversation_id):
        # Atomic increment
        return self.redis.incr(f"seq:{conversation_id}")

class ChatService:
    def send_message(self, sender_id, conversation_id, content):
        # Get sequence number
        sequence = self.sequencer.get_next_sequence(conversation_id)

        message = {
            "id": generate_id(),
            "sequence": sequence,
            "sender_id": sender_id,
            "content": content,
        }

        self.message_store.save(message)
        self.deliver(message)

class ChatClient:
    def on_message(self, message):
        expected_seq = self.last_sequence[message.conversation_id] + 1

        if message.sequence == expected_seq:
            self.display(message)
            self.last_sequence[message.conversation_id] = message.sequence
        elif message.sequence > expected_seq:
            # Missing messages, fetch from server
            self.fetch_missing(message.conversation_id, expected_seq, message.sequence)

class PresenceService:
    def __init__(self):
        self.presence_cache = Redis()
        self.heartbeat_interval = 30  # seconds

    def set_online(self, user_id):
        self.presence_cache.setex(
            f"presence:{user_id}",
            self.heartbeat_interval * 2,
            "online"
        )
        self.publish_presence_change(user_id, "online")

    def heartbeat(self, user_id):
        self.presence_cache.expire(
            f"presence:{user_id}",
            self.heartbeat_interval * 2
        )

    def is_online(self, user_id):
        return self.presence_cache.exists(f"presence:{user_id}")

    def get_online_friends(self, user_id):
        friends = self.social_graph.get_friends(user_id)

        # Batch check presence
        pipeline = self.presence_cache.pipeline()
        for friend_id in friends:
            pipeline.exists(f"presence:{friend_id}")

        results = pipeline.execute()

        return [f for f, online in zip(friends, results) if online]

Challenge:
- 50M users online
- Each has ~200 friends average
- If everyone checks friends every 30 seconds:
  50M × 200 / 30 = 333M presence checks/second

Solution: Presence grouping

class ScalablePresence:
    def __init__(self):
        self.presence_buckets = 10000  # Partition presence data

    def get_bucket(self, user_id):
        return hash(user_id) % self.presence_buckets

    def subscribe_to_friends(self, user_id, friend_ids):
        # Group friends by bucket
        buckets = defaultdict(list)
        for friend_id in friend_ids:
            bucket = self.get_bucket(friend_id)
            buckets[bucket].append(friend_id)

        # Subscribe to relevant buckets only
        for bucket in buckets.keys():
            self.pubsub.subscribe(f"presence_bucket:{bucket}")

    def on_presence_change(self, user_id, status):
        bucket = self.get_bucket(user_id)

        # Publish to bucket
        self.pubsub.publish(
            f"presence_bucket:{bucket}",
            {"user_id": user_id, "status": status}
        )

Feed: Hybrid fanout (push + pull)
Payments: Multiple providers
Chat: WebSocket + HTTP fallback

Pure approaches rarely work at scale.
Combine strategies based on access patterns.

Feed: Eventual (acceptable)
Payments: Strong (required)
Chat: Eventual with ordering guarantees

Choose consistency based on business requirements,
not technical preference.

Feed: Async fanout
Payments: Sync for charge, async for notifications
Chat: Async delivery, sync for send acknowledgment

Move work out of critical path when possible.

Feed: Multi-tier caching
Payments: Idempotency cache
Chat: Presence cache, message cache

Caching is not optimization.
It is architecture requirement at scale.

Feed: Duplicate post detection
Payments: Idempotency keys
Chat: Message deduplication

At scale, every operation will be retried.
Make everything idempotent.

All systems need:
- Request tracing
- Error tracking
- Performance metrics
- Business metrics

You cannot operate what you cannot observe.