How to Build an AI Receptionist: Step-by-Step Guide 2026

# docker-compose.yml - Development Environment
version: '3.8'
services:
  api:
    build: ./api
    ports:
      - "3000:3000"
    environment:
      - DATABASE_URL=postgresql://postgres:password@db:5432/voiceai
      - REDIS_URL=redis://redis:6379
      - TWILIO_ACCOUNT_SID=your_account_sid
      - TWILIO_AUTH_TOKEN=your_auth_token
      - OPENAI_API_KEY=your_openai_key
      - DEEPGRAM_API_KEY=your_deepgram_key
      - ELEVENLABS_API_KEY=your_elevenlabs_key
    depends_on:
      - db
      - redis

  db:
    image: postgres:15
    environment:
      POSTGRES_DB: voiceai
      POSTGRES_USER: postgres
      POSTGRES_PASSWORD: password
    ports:
      - "5432:5432"
    volumes:
      - postgres_data:/var/lib/postgresql/data

  redis:
    image: redis:7-alpine
    ports:
      - "6379:6379"

  ngrok:
    image: ngrok/ngrok:latest
    restart: unless-stopped
    command:
      - "start"
      - "--all"
      - "--config"
      - "/etc/ngrok.yml"
    volumes:
      - ./ngrok.yml:/etc/ngrok.yml
    ports:
      - 4040:4040

volumes:
  postgres_data:

// Basic Twilio webhook handler
import express from 'express';
import { VoiceResponse } from 'twilio/lib/twiml/VoiceResponse';

const app = express();

app.post('/webhook/incoming-call', async (req, res) => {
  const twiml = new VoiceResponse();
  
  // Start recording for quality assurance
  twiml.record({
    action: '/webhook/recording-complete',
    method: 'POST',
    maxLength: 3600, // 1 hour max
    playBeep: false
  });
  
  // Initial greeting
  twiml.say({
    voice: 'Polly.Joanna',
    language: 'en-US'
  }, 'Hello! I\'m your AI assistant. How can I help you today?');
  
  // Start listening for speech
  twiml.gather({
    input: ['speech'],
    timeout: 5,
    speechTimeout: 'auto',
    action: '/webhook/process-speech',
    method: 'POST'
  });
  
  // Fallback if no input
  twiml.say('I didn\'t hear anything. Please let me know how I can help you.');
  twiml.hangup();
  
  res.type('text/xml');
  res.send(twiml.toString());
});

app.post('/webhook/process-speech', async (req, res) => {
  const { SpeechResult, CallSid, From } = req.body;
  
  try {
    // Process the speech with AI
    const aiResponse = await processConversation({
      callSid: CallSid,
      callerNumber: From,
      userInput: SpeechResult
    });
    
    const twiml = new VoiceResponse();
    
    // Handle different response types
    switch (aiResponse.action) {
      case 'respond':
        twiml.say({
          voice: 'Polly.Joanna'
        }, aiResponse.message);
        
        // Continue conversation
        twiml.gather({
          input: ['speech'],
          timeout: 5,
          speechTimeout: 'auto',
          action: '/webhook/process-speech',
          method: 'POST'
        });
        break;
        
      case 'transfer':
        twiml.say('Let me transfer you to the right person.');
        twiml.dial(aiResponse.transferNumber);
        break;
        
      case 'hangup':
        twiml.say(aiResponse.message);
        twiml.hangup();
        break;
        
      default:
        twiml.say('I\'m having trouble understanding. Let me get you to a person who can help.');
        twiml.dial(process.env.FALLBACK_NUMBER);
    }
    
    res.type('text/xml');
    res.send(twiml.toString());
    
  } catch (error) {
    console.error('Error processing speech:', error);
    
    // Graceful fallback
    const twiml = new VoiceResponse();
    twiml.say('I\'m experiencing technical difficulties. Let me connect you with someone who can help.');
    twiml.dial(process.env.FALLBACK_NUMBER);
    
    res.type('text/xml');
    res.send(twiml.toString());
  }
});

// Deepgram streaming speech recognition
import { createClient } from '@deepgram/sdk';
import WebSocket from 'ws';

class SpeechProcessor {
  constructor(callSid) {
    this.callSid = callSid;
    this.deepgram = createClient(process.env.DEEPGRAM_API_KEY);
    this.conversationContext = [];
  }

  async startStreaming(twilioStream) {
    const connection = this.deepgram.listen.live({
      model: 'nova-2',
      language: 'en-US',
      smart_format: true,
      interim_results: true,
      utterance_end_ms: 1000,
      endpointing: 300,
      channels: 1,
      sample_rate: 8000
    });

    connection.on('open', () => {
      console.log(`Speech recognition started for call ${this.callSid}`);
    });

    connection.on('results', async (data) => {
      const transcript = data.channel.alternatives[0];
      
      // Only process final results with high confidence
      if (transcript.confidence > 0.7 && data.is_final) {
        console.log(`Final transcript: ${transcript.transcript}`);
        
        // Process with AI conversation engine
        const response = await this.processWithAI(transcript.transcript);
        
        // Send response back to Twilio
        await this.sendResponseToTwilio(response);
      }
    });

    connection.on('error', (error) => {
      console.error('Deepgram error:', error);
      // Implement fallback or retry logic
    });

    // Forward audio from Twilio to Deepgram
    twilioStream.on('media', (payload) => {
      const audioBuffer = Buffer.from(payload.media.payload, 'base64');
      connection.send(audioBuffer);
    });

    twilioStream.on('stop', () => {
      connection.finish();
    });

    return connection;
  }

  async processWithAI(transcript) {
    // Add to conversation context
    this.conversationContext.push({
      role: 'user',
      content: transcript,
      timestamp: new Date()
    });

    // Call your AI conversation engine here
    return await this.generateAIResponse(this.conversationContext);
  }
}

// AI Conversation Engine
class ConversationEngine {
  constructor(businessConfig) {
    this.businessConfig = businessConfig;
    this.openai = new OpenAI({
      apiKey: process.env.OPENAI_API_KEY
    });
  }

  async processConversation({ callSid, callerNumber, userInput, context = [] }) {
    // Build conversation context with business information
    const systemPrompt = this.buildSystemPrompt();
    const conversationHistory = this.buildConversationHistory(context);
    
    const messages = [
      { role: 'system', content: systemPrompt },
      ...conversationHistory,
      { role: 'user', content: userInput }
    ];

    try {
      const response = await this.openai.chat.completions.create({
        model: 'gpt-4',
        messages,
        functions: this.getAvailableFunctions(),
        function_call: 'auto',
        temperature: 0.3, // Lower temperature for more consistent responses
        max_tokens: 200   // Keep responses concise for voice
      });

      const aiMessage = response.choices[0].message;

      // Handle function calls (booking, lookups, etc.)
      if (aiMessage.function_call) {
        return await this.handleFunctionCall(aiMessage.function_call, callSid);
      }

      // Regular text response
      return {
        action: 'respond',
        message: this.optimizeForSpeech(aiMessage.content),
        shouldContinue: this.shouldContinueConversation(aiMessage.content)
      };

    } catch (error) {
      console.error('AI processing error:', error);
      return {
        action: 'transfer',
        message: 'Let me connect you with someone who can help you right away.',
        transferNumber: this.businessConfig.fallbackNumber
      };
    }
  }

  buildSystemPrompt() {
    const { businessName, businessType, services, hours, policies } = this.businessConfig;
    
    return `You are the AI receptionist for ${businessName}, a ${businessType} business.

BUSINESS INFORMATION:
- Services: ${services.join(', ')}
- Hours: ${hours}
- Emergency policy: ${policies.emergency}

PERSONALITY:
- Professional but friendly
- Helpful and solution-oriented  
- Knowledgeable about our services
- Can make appointments and provide information

CONVERSATION RULES:
1. Keep responses under 30 words when possible
2. Always confirm important details (appointments, contact info)
3. If you can't help, offer to transfer to a human
4. For emergencies, gather location and contact info immediately
5. Use natural speech patterns, avoid robotic responses

AVAILABLE ACTIONS:
- book_appointment: Schedule service appointments
- check_availability: Look up available time slots
- get_pricing: Provide service pricing
- transfer_call: Connect to human agent
- schedule_callback: Arrange for someone to call back

Remember: You're representing our business. Be professional, helpful, and make sure customers feel heard.`;
  }

  getAvailableFunctions() {
    return [
      {
        name: 'book_appointment',
        description: 'Book an appointment for the customer',
        parameters: {
          type: 'object',
          properties: {
            service_type: { type: 'string' },
            preferred_date: { type: 'string' },
            preferred_time: { type: 'string' },
            customer_name: { type: 'string' },
            customer_phone: { type: 'string' },
            customer_address: { type: 'string' },
            urgency: { type: 'string', enum: ['routine', 'urgent', 'emergency'] }
          },
          required: ['service_type', 'customer_name', 'customer_phone']
        }
      },
      {
        name: 'check_availability',
        description: 'Check available appointment slots',
        parameters: {
          type: 'object',
          properties: {
            date: { type: 'string' },
            service_duration: { type: 'number' }
          }
        }
      },
      {
        name: 'transfer_call',
        description: 'Transfer call to human agent',
        parameters: {
          type: 'object',
          properties: {
            reason: { type: 'string' },
            urgency: { type: 'string', enum: ['low', 'medium', 'high'] }
          }
        }
      }
    ];
  }

  async handleFunctionCall(functionCall, callSid) {
    const { name, arguments: args } = functionCall;
    const parsedArgs = JSON.parse(args);

    switch (name) {
      case 'book_appointment':
        return await this.bookAppointment(parsedArgs, callSid);
      
      case 'check_availability':
        return await this.checkAvailability(parsedArgs);
      
      case 'transfer_call':
        return {
          action: 'transfer',
          message: 'Let me connect you with one of our team members.',
          transferNumber: this.getTransferNumber(parsedArgs.urgency)
        };
      
      default:
        return {
          action: 'respond',
          message: 'Let me look that up for you and get back to you.',
          shouldContinue: true
        };
    }
  }

  optimizeForSpeech(text) {
    return text
      .replace(/([0-9]+)/g, (match) => this.numberToWords(match))
      .replace(/&/g, 'and')
      .replace(/$/g, 'dollar')
      .replace(/%/g, 'percent');
  }
}