AI Phone Ordering Agent for Restaurants: Taking Food Orders via Voice

The Phone Ordering Problem in Restaurants

Phone orders account for 30 to 50 percent of revenue at many takeout and delivery restaurants, yet handling them is painful. Staff get pulled away from in-house guests, orders are misheard, and peak-hour calls go unanswered. An AI phone ordering agent solves this by converting spoken requests into structured orders with perfect accuracy and infinite patience.

The challenge is not speech recognition alone — it is building an agent that understands menu semantics, handles customizations like "extra cheese, no onions, make it spicy," confirms totals, and pushes the final order into the restaurant's POS system.

Structuring the Menu for Agent Consumption

The agent needs a machine-readable menu model that captures items, modifiers, pricing, and constraints.

sequenceDiagram
    autonumber
    participant Caller as Caller
    participant Agent as CallSphere Agent
    participant API as CRM API
    participant DB as CRM Database
    participant Webhook as Webhook Listener
    Caller->>Agent: Inbound call begins
    Agent->>Agent: STT plus intent detection
    Agent->>API: Lookup contact by phone
    API->>DB: Read contact record
    DB-->>API: Contact and history
    API-->>Agent: Personalized context
    Agent->>API: Create call activity
    Agent->>API: Update deal stage
    API->>Webhook: Outbound webhook fires
    Webhook-->>Agent: Confirmed
    Agent->>Caller: Spoken confirmation

from dataclasses import dataclass, field
from typing import Optional

@dataclass
class Modifier:
    name: str
    price_delta: float = 0.0
    category: str = "addon"  # addon, removal, substitution, size

@dataclass
class MenuItem:
    item_id: str
    name: str
    base_price: float
    category: str
    description: str
    available_modifiers: list[Modifier] = field(default_factory=list)
    available: bool = True

@dataclass
class OrderItem:
    menu_item: MenuItem
    quantity: int
    modifiers: list[Modifier] = field(default_factory=list)
    special_instructions: str = ""

    @property
    def line_total(self) -> float:
        modifier_cost = sum(m.price_delta for m in self.modifiers)
        return (self.menu_item.base_price + modifier_cost) * self.quantity

@dataclass
class Order:
    items: list[OrderItem] = field(default_factory=list)
    customer_name: str = ""
    customer_phone: str = ""
    order_type: str = "pickup"  # pickup, delivery

    @property
    def subtotal(self) -> float:
        return sum(item.line_total for item in self.items)

    @property
    def tax(self) -> float:
        return round(self.subtotal * 0.0875, 2)

    @property
    def total(self) -> float:
        return self.subtotal + self.tax

    def summary(self) -> str:
        lines = []
        for item in self.items:
            mods = ", ".join(m.name for m in item.modifiers)
            mod_str = f" ({mods})" if mods else ""
            lines.append(
                f"  {item.quantity}x {item.menu_item.name}{mod_str}"
                f" - ${item.line_total:.2f}"
            )
        lines.append(f"  Subtotal: ${self.subtotal:.2f}")
        lines.append(f"  Tax: ${self.tax:.2f}")
        lines.append(f"  Total: ${self.total:.2f}")
        return "\n".join(lines)

Building the Ordering Agent Tools

The agent needs tools to search the menu, add items, apply modifiers, and finalize orders.

from agents import Agent, function_tool

menu_items = [
    MenuItem("B1", "Classic Burger", 12.99, "Burgers",
             "Beef patty with lettuce and tomato",
             [Modifier("Extra Cheese", 1.50), Modifier("No Onions"),
              Modifier("Add Bacon", 2.00), Modifier("Make it Spicy")]),
    MenuItem("P1", "Margherita Pizza", 14.99, "Pizza",
             "Fresh mozzarella and basil on tomato sauce",
             [Modifier("Large Size", 4.00, "size"),
              Modifier("Extra Cheese", 2.00), Modifier("Add Pepperoni", 2.50)]),
    MenuItem("S1", "Caesar Salad", 9.99, "Salads",
             "Romaine, parmesan, croutons, caesar dressing",
             [Modifier("Add Grilled Chicken", 4.00),
              Modifier("No Croutons", 0.0, "removal")]),
]

current_order = Order()

@function_tool
def search_menu(query: str) -> str:
    query_lower = query.lower()
    matches = [
        item for item in menu_items
        if query_lower in item.name.lower()
        or query_lower in item.category.lower()
        or query_lower in item.description.lower()
    ]
    if not matches:
        return f"No menu items matching '{query}'."
    lines = [f"- {m.name} (${m.base_price:.2f}): {m.description}" for m in matches]
    return "\n".join(lines)

@function_tool
def add_to_order(
    item_id: str, quantity: int, modifier_names: list[str],
    special_instructions: str = ""
) -> str:
    menu_item = next((m for m in menu_items if m.item_id == item_id), None)
    if not menu_item:
        return f"Item {item_id} not found on menu."
    if not menu_item.available:
        return f"{menu_item.name} is currently unavailable."
    selected_mods = [
        m for m in menu_item.available_modifiers
        if m.name.lower() in [n.lower() for n in modifier_names]
    ]
    order_item = OrderItem(menu_item, quantity, selected_mods, special_instructions)
    current_order.items.append(order_item)
    return f"Added {quantity}x {menu_item.name} to order. Running total: ${current_order.total:.2f}"

@function_tool
def get_order_summary() -> str:
    if not current_order.items:
        return "The order is currently empty."
    return current_order.summary()

@function_tool
def finalize_order(customer_name: str, customer_phone: str, order_type: str) -> str:
    if not current_order.items:
        return "Cannot finalize an empty order."
    current_order.customer_name = customer_name
    current_order.customer_phone = customer_phone
    current_order.order_type = order_type
    return (
        f"Order confirmed for {customer_name} ({order_type}). "
        f"Total: ${current_order.total:.2f}. "
        f"Estimated ready time: 25-30 minutes."
    )

POS Integration Pattern

The final step is pushing confirmed orders into the restaurant's point-of-sale system. Most modern POS systems expose REST APIs.

import httpx

async def push_to_pos(order: Order, pos_api_url: str, api_key: str) -> dict:
    payload = {
        "customer": {
            "name": order.customer_name,
            "phone": order.customer_phone,
        },
        "type": order.order_type,
        "items": [
            {
                "sku": item.menu_item.item_id,
                "name": item.menu_item.name,
                "quantity": item.quantity,
                "modifiers": [m.name for m in item.modifiers],
                "special_instructions": item.special_instructions,
                "line_total": item.line_total,
            }
            for item in order.items
        ],
        "subtotal": order.subtotal,
        "tax": order.tax,
        "total": order.total,
    }
    async with httpx.AsyncClient() as client:
        response = await client.post(
            f"{pos_api_url}/orders",
            json=payload,
            headers={"Authorization": f"Bearer {api_key}"},
        )
        response.raise_for_status()
        return response.json()

Wiring the Agent

ordering_agent = Agent(
    name="Phone Ordering Agent",
    instructions="""You are a friendly phone ordering agent for a restaurant.
    Guide callers through the menu, take their order with any customizations,
    read back the complete order for confirmation, then finalize it.
    Always confirm the total before finalizing. Be patient with modifications.""",
    tools=[search_menu, add_to_order, get_order_summary, finalize_order],
)

FAQ

How does the agent handle ambiguous voice input like "the usual" or "same as last time"?

The agent integrates with a customer profile database keyed by phone number. When a returning caller is identified via caller ID, the agent retrieves their order history and can suggest or replicate previous orders. For first-time callers, it gracefully asks the customer to specify their order.

What happens when an item is out of stock mid-conversation?

Menu item availability is checked at the moment add_to_order is called, not when the menu is browsed. If an item becomes unavailable between browsing and ordering, the tool returns an unavailability message and the agent suggests similar alternatives from the same category.

How do you handle complex modifier combinations that are invalid?

The menu model can be extended with a modifier_rules field that defines exclusion groups (for example, you cannot select both "no cheese" and "extra cheese"). The add_to_order function validates modifier combinations against these rules before accepting the order line item.

---

#VoiceAI #RestaurantOrdering #POSIntegration #AgenticAI #Python #LearnAI #AIEngineering