ESP32 Edge AI: Compressed Models for DIY Projects

[ESP32] ── SDA ── [I2C LCD Display]
[ESP32] ── SCL ──
[ESP32] ── MIC ── [Optional I2S Microphone]
[ESP32] ── GND ── [All components GND]
[ESP32] ── 3.3V ── [All components 3.3V]

# Python script to convert your trained model
import tensorflow as tf
import numpy as np

# Load your trained model
model = tf.keras.models.load_model('your_voice_model.h5')

# Convert to TensorFlow Lite format
converter = tf.lite.TFLiteConverter.from_keras_model(model)
converter.optimizations = [tf.lite.Optimize.DEFAULT]
converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]
converter.inference_input_type = tf.float32
converter.inference_output_type = tf.float32

# Convert the model
tflite_model = converter.convert()

# Save the model
with open('compressed_model.tflite', 'wb') as f:
    f.write(tflite_model)

print('Model conversion complete!')

#include <Arduino.h>
#include <TensorFlowLite_ESP32.h>
#include <tensorflow/lite/micro/all_ops_resolver.h>
#include <tensorflow/lite/micro/micro_error_reporter.h>
#include <tensorflow/lite/micro/micro_interpreter.h>
#include <tensorflow/lite/schema/schema_generated.h>

#include "I2C_LCD_Arduino.h"

// Model and interpreter
const tflite::Model* model = nullptr;
tflite::MicroErrorReporter micro_error_reporter;
tflite::AllOpsResolver resolver;
tflite::MicroInterpreter* interpreter = nullptr;
TfLiteTensor* input_tensor = nullptr;
TfLiteTensor* output_tensor = nullptr;

// LCD object
I2C_LCD lcd(0x27, 16, 2);

// Audio buffer
const int AUDIO_BUFFER_SIZE = 1024;
int16_t audio_buffer[AUDIO_BUFFER_SIZE];
int audio_index = 0;

void setup() {
  Serial.begin(115200);
  lcd.begin();
  lcd.backlight();
  lcd.setCursor(0, 0);
  lcd.print("Initializing...");
  
  // Load the model
  loadModel();
  
  // Initialize audio input
  setupAudio();
  
  delay(2000);
  lcd.clear();
  lcd.setCursor(0, 0);
  lcd.print("Voice Command");
  lcd.setCursor(0, 1);
  lcd.print("Ready!");
}

void loop() {
  // Check for voice input
  if (checkForVoiceInput()) {
    // Preprocess audio
    preprocessAudio();
    
    // Run inference
    runInference();
    
    // Display result
    displayResult();
  }
  
  delay(50);
}

void loadModel() {
  // Read model from SD card
  File modelFile = SD.open("compressed_model.tflite", FILE_READ);
  if (!modelFile) {
    lcd.clear();
    lcd.print("Model load failed");
    return;
  }
  
  f_size = modelFile.size();
  model_data = (uint8_t*)malloc(f_size);
  if (model_data == nullptr) {
    lcd.clear();
    lcd.print("Memory allocation failed");
    return;
  }
  
  modelFile.read(model_data, f_size);
  modelFile.close();
  
  // Load the TFLite model
  model = tflite::GetModel(model_data);
  if (model->version() != TFLITE_SCHEMA_VERSION) {
    lcd.clear();
    lcd.print("Model schema error");
    return;
  }
  
  // Create interpreter
  static tflite::MicroInterpreter static_interpreter(
      model, resolver, nullptr, &micro_error_reporter);
  interpreter = &static_interpreter;
  
  // Allocate tensor buffers
  TfLiteStatus allocate_status = interpreter->AllocateTensors();
  if (allocate_status != kTfLiteOk) {
    lcd.clear();
    lcd.print("Buffer allocation failed");
    return;
  }
  
  // Get input and output tensors
  input_tensor = interpreter->input(0);
  output_tensor = interpreter->output(0);
  
  if (input_tensor == nullptr || output_tensor == nullptr) {
    lcd.clear();
    lcd.print("Tensor allocation failed");
    return;
  }
  
  lcd.clear();
  lcd.print("Model loaded!");
  delay(2000);
}

void setupAudio() {
  // Initialize audio sampling
  audio_index = 0;
  memset(audio_buffer, 0, sizeof(audio_buffer));
}

bool checkForVoiceInput() {
  // Simple voice detection based on amplitude
  uint16_t sum = 0;
  for (int i = 0; i < AUDIO_BUFFER_SIZE; i++) {
    sum += abs(audio_buffer[i]);
  }
  
  uint16_t average = sum / AUDIO_BUFFER_SIZE;
  return average > 100; // Threshold for voice detection
}

void preprocessAudio() {
  // Apply preprocessing steps
  // 1. Normalize audio
  for (int i = 0; i < AUDIO_BUFFER_SIZE; i++) {
    audio_buffer[i] = audio_buffer[i] / 32768.0 * 32767;
  }
  
  // 2. Apply window function (Hann window)
  for (int i = 0; i < AUDIO_BUFFER_SIZE; i++) {
    float window = 0.5 * (1 - cos(2 * PI * i / (AUDIO_BUFFER_SIZE - 1)));
    audio_buffer[i] = (int16_t)(audio_buffer[i] * window);
  }
  
  // 3. Convert to float and reshape for model input
  float input_data[1, AUDIO_BUFFER_SIZE];
  for (int i = 0; i < AUDIO_BUFFER_SIZE; i++) {
    input_data[0][i] = audio_buffer[i] / 32768.0;
  }
  
  // Set tensor data
  memcpy(interpreter->input(0)->data.raw, input_data, sizeof(input_data));
}

void runInference() {
  // Invoke the interpreter
  TfLiteStatus invoke_status = interpreter->Invoke();
  
  if (invoke_status != kTfLiteOk) {
    lcd.clear();
    lcd.print("Inference failed");
    return;
  }
}

void displayResult() {
  // Get output scores
  float* output_data = interpreter->output(0)->data.f;
  
  // Find the command with highest probability
  int max_index = 0;
  float max_value = output_data[0];
  
  for (int i = 1; i < 9; i++) {
    if (output_data[i] > max_value) {
      max_value = output_data[i];
      max_index = i;
    }
  }
  
  // Display result on LCD
  lcd.clear();
  lcd.setCursor(0, 0);
  lcd.print("Command detected");
  
  String command = getCommandName(max_index);
  lcd.setCursor(0, 1);
  lcd.print(command);
  
  // Print probabilities for debugging
  Serial.print("Command: ");
  Serial.print(command);
  Serial.print(" Confidence: ");
  Serial.print(max_value * 100, 2);
  Serial.println("%");
  
  delay(2000);
}

String getCommandName(int index) {
  const String commands[] = {
    "UNKNOWN", "HELLO", "GOODBYE", "LIGHT ON", 
    "LIGHT OFF", "STOP", "START", "UP", "DOWN"
  };
  
  if (index >= 0 && index < 9) {
    return commands[index];
  }
  return "UNKNOWN";
}