I'm trying to get a sense of the rate limitations of a python frontend. I 
understand this will vary from system to system.

I adapted two frontends from the example templates, one in C++ and one in python. 
Both simply fill a midas bank with a fixed length array of zeros at a given polled 
rate. However, the C++ frontend is about 100 times faster in both data and event 
rates. This seems slow, even for an interpreted language like python. Furthermore, 
I can effectively increase the maximum rate by concurrently running a second 
python frontend (this is not the case for the C++ frontend). In short, there is 
some limitation with using python here unrelated to hardware.

In my case, poll_func appears to be called at 100Hz at best. What limits the rate 
that poll_func is called in a python frontend? Is there a more appropriate 
solution for increasing the python frontend data/event rate than simply launching 
more frontends?

I've attached my C++ and python frontend files for reference.

Thanks,
Jack

import midas
import midas.frontend
import midas.event
import numpy as np
import random
import time

class DataSimulatorEquipment(midas.frontend.EquipmentBase):
    def __init__(self, client, frontend):
        equip_name = "Python Data Simulator"
        default_common = midas.frontend.InitialEquipmentCommon()
        default_common.equip_type = midas.EQ_POLLED
        default_common.buffer_name = "SYSTEM"
        default_common.trigger_mask = 0
        default_common.event_id = 2
        default_common.period_ms = 100
        default_common.read_when = midas.RO_RUNNING
        default_common.log_history = 1
        
        midas.frontend.EquipmentBase.__init__(self, client, equip_name, default_common)
        print("Initialization complete")
        self.set_status("Initialized")

        self.frontend = frontend

    def readout_func(self):
        event = midas.event.Event()
        
        # Create a bank for zero buffer
        event.create_bank("CR00", midas.TID_SHORT, self.frontend.zero_buffer)
        
        # Simulate the addition of `data` in the periodic event
        '''
        data_block = []
        data_block.extend(self.frontend.data)
        
        
        # Append the simulated data to the event
        event.create_bank("CR00", midas.TID_SHORT, data_block)
        '''

        return event
    
    def poll_func(self):
        current_time = time.time()
        if current_time - self.frontend.last_poll_time >= self.frontend.poll_time:
            self.frontend.last_poll_time = current_time
            self.frontend.poll_count += 1
            self.frontend.poll_timestamps.append(current_time)
            return True  # Indicate that an event is available
        return False  # No event available yet

class DataSimulatorFrontend(midas.frontend.FrontendBase):
    def __init__(self):
        midas.frontend.FrontendBase.__init__(self, "DataSimulator-Python")
        
        # Data and zero buffer initialization
        self.data = []
        self.zero_buffer = []
        self.generator = random.Random()
        self.total_data_size = 1250000
        self.load_data_from_file("fake_data.txt")
        self.init_zero_buffer()

        # Polling variables
        self.poll_time = 0.001  # Poll time in seconds
        self.last_poll_time = time.time()
        self.poll_count = 0
        self.poll_timestamps = []

        self.add_equipment(DataSimulatorEquipment(self.client, self))

    def load_data_from_file(self, filename):
        try:
            with open(filename, 'r') as file:
                for line in file:
                    values = [int(value) for value in line.strip().split(',')]
                    self.data.extend(values)
            print(f"Loaded data from {filename}: {self.data[:10]}...")  # Display the first few values for verification
        except IOError as e:
            print(f"Error opening file: {e}")

    def init_zero_buffer(self):
        self.zero_buffer = [0] * self.total_data_size 
        print(f"Initialized zero buffer with {self.total_data_size } zeros.")

    def begin_of_run(self, run_number):
        self.set_all_equipment_status("Running", "greenLight")
        self.client.msg(f"Frontend has started run number {run_number}")
        return midas.status_codes["SUCCESS"]

    def end_of_run(self, run_number):
        self.set_all_equipment_status("Finished", "greenLight")
        self.client.msg(f"Frontend has ended run number {run_number}")
        
        # Print poll function statistics at the end of the run
        self.print_poll_stats()

        return midas.status_codes["SUCCESS"]

    def frontend_exit(self):
        print("Frontend is exiting.")

    def print_poll_stats(self):
        if len(self.poll_timestamps) > 1:
            intervals = [self.poll_timestamps[i] - self.poll_timestamps[i-1] for i in range(1, len(self.poll_timestamps))]
            avg_interval = sum(intervals) / len(intervals)
            print(f"Poll function was called {self.poll_count} times.")
            print(f"Average interval between poll calls: {avg_interval:.6f} seconds")
        else:
            print(f"Poll function was called {self.poll_count} times. Not enough data for interval calculation.")

if __name__ == "__main__":
    with DataSimulatorFrontend() as my_fe:
        my_fe.run()

#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <string.h>
#include <iostream>
#include <fstream>
#include <sstream>
#include <vector>
#include "midas.h"
#include "mfe.h"
#include <stdlib.h> // Include the header for rand()
#include <random> // Include for random number generation


void trigger_update(INT, INT, void*);

/*-- Globals -------------------------------------------------------*/

/* The frontend name (client name) as seen by other MIDAS clients   */
const char *frontend_name = "DataSimulator";
/* The frontend file name, don't change it */
const char *frontend_file_name = __FILE__;

/* frontend_loop is called periodically if this variable is TRUE    */
BOOL frontend_call_loop = FALSE;

/* a frontend status page is displayed with this frequency in ms */
INT display_period = 1000;

/* maximum event size produced by this frontend */
INT max_event_size = 1024 * 1014;

/* maximum event size for fragmented events (EQ_FRAGMENTED) */
INT max_event_size_frag = 5 * max_event_size;

/* buffer size to hold events */
INT event_buffer_size = 5 * max_event_size;

// Define a vector to store 16-bit words
std::vector<int16_t> data; // Define a global vector to store 16-bit signed integers

// Global variable to keep track of the last poll time
std::chrono::steady_clock::time_point last_poll_time;
const std::chrono::microseconds polling_interval(300); // Poll every 300 microsecond

// Random number generator for generating data
std::mt19937 generator;
std::uniform_int_distribution<short> distribution(-32768, 32767); // Define the range of random values (short range)

// Global variable to hold the zero buffer
std::vector<short> zero_buffer;


/*-- Function declarations -----------------------------------------*/

INT frontend_init(void);
INT frontend_exit(void);
INT begin_of_run(INT run_number, char *error);
INT end_of_run(INT run_number, char *error);
INT pause_run(INT run_number, char *error);
INT resume_run(INT run_number, char *error);
INT frontend_loop(void);

INT read_trigger_event(char *pevent, INT off);
INT read_periodic_event(char *pevent, INT off);

INT poll_event(INT source, INT count, BOOL test);
INT interrupt_configure(INT cmd, INT source, POINTER_T adr);

/*-- Equipment list ------------------------------------------------*/

BOOL equipment_common_overwrite = TRUE;

EQUIPMENT equipment[] = {
   {"Data Simulator",              /* equipment name */
      {2, 0,                 /* event ID, trigger mask */
         "SYSTEM",           /* event buffer */
         EQ_POLLED,        /* equipment type */
         0,                  /* event source */
         "MIDAS",            /* format */
         TRUE,               /* enabled */
         RO_RUNNING | RO_TRANSITIONS |   /* read when running and on transitions */
         RO_ODB,             /* and update ODB */
         10,               /* read every sec */
         0,                  /* stop run after this event limit */
         0,                  /* number of sub events */
         TRUE,               /* log history */
         "", "", "",},
      read_trigger_event   /* readout routine */
   },

   {""}
};

/*-- Trigger Update ------------------------------------------------*/

void trigger_update(INT hDB, INT hkey,void*)
{

}


/*-- Frontend Init -------------------------------------------------*/

int frontend_init() {
    // Open the file for reading
    std::ifstream inputFile("fake_data.txt");

    if (!inputFile) {
        std::cerr << "Error opening the file." << std::endl;
        return 1;
    }

    std::cout << "Reading and converting data:" << std::endl;

    std::string line;
    while (std::getline(inputFile, line)) {
        std::istringstream iss(line);
        std::string token;

        while (std::getline(iss, token, ',')) {
            int16_t value;
            std::istringstream(token) >> value;
            data.push_back(value);
        }
    }

    // Print the converted data
    for (int i = 0; i < data.size(); i++) {
        std::cout << " " << data[i];
    }

    // Close the file
    inputFile.close();

    if (data.empty()) {
        std::cerr << "No data was converted." << std::endl;
    } else {
        std::cout << std::endl << "Conversion completed." << std::endl;
    }

    // Initialize random number generator
    std::random_device rd; // Obtain a random number from hardware
    generator = std::mt19937(rd()); // Seed the generator

    // Define the total number of zero data points
    const int total_data_size = 50000; // Adjust size as needed

    // Create and initialize the buffer of zeros
    zero_buffer.resize(total_data_size, 0);

    return SUCCESS;
}




/*-- Frontend Exit -------------------------------------------------*/

INT frontend_exit()
{
   return SUCCESS;
}

/*-- Begin of Run --------------------------------------------------*/

INT begin_of_run(INT run_number, char *error)
{
   return SUCCESS;
}

/*-- End of Run ----------------------------------------------------*/

INT end_of_run(INT run_number, char *error)
{
   return SUCCESS;
}

/*-- Pause Run -----------------------------------------------------*/

INT pause_run(INT run_number, char *error)
{
   return SUCCESS;
}

/*-- Resume Run ----------------------------------------------------*/

INT resume_run(INT run_number, char *error)
{
   return SUCCESS;
}

/*-- Frontend Loop -------------------------------------------------*/

INT frontend_loop()
{
   /* if frontend_call_loop is true, this routine gets called when
      the frontend is idle or once between every event */
   return SUCCESS;
}

/*------------------------------------------------------------------*/

/********************************************************************\

  Readout routines for different events

\********************************************************************/

/*-- Trigger event routines ----------------------------------------*/

INT poll_event(INT source, INT count, BOOL test) {
    // Get the current time
    auto now = std::chrono::steady_clock::now();
    
    // Check if enough time has passed since the last poll
    if (now - last_poll_time >= polling_interval) {
        // Update the last poll time
        last_poll_time = now;
        
        // Return TRUE to indicate that an event is available
        return TRUE;
    }
    
    // If test is TRUE, don't return anything
    if (test) {
        return FALSE;
    }
    
    // Otherwise, return FALSE to indicate no event available
    return FALSE;
}

/*-- Interrupt configuration ---------------------------------------*/

INT interrupt_configure(INT cmd, INT source, POINTER_T adr)
{
   switch (cmd) {
   case CMD_INTERRUPT_ENABLE:
      break;
   case CMD_INTERRUPT_DISABLE:
      break;
   case CMD_INTERRUPT_ATTACH:
      break;
   case CMD_INTERRUPT_DETACH:
      break;
   }
   return SUCCESS;
}

/*-- Event readout -------------------------------------------------*/

INT read_trigger_event(char *pevent, INT off)
{
    short *pdata;

    // Init bank structure
    bk_init32(pevent);

    // Create a bank named "CR00" and specify the data type as TID_SHORT
    bk_create(pevent, "CR00", TID_SHORT, (void **)&pdata);

    // Use memcpy to copy the buffer of zeros into the MIDAS bank
    memcpy(pdata, zero_buffer.data(), zero_buffer.size() * sizeof(short));

    // Adjust pdata pointer
    pdata += zero_buffer.size();  // Move the pointer past the copied data

    // Close the bank
    bk_close(pevent, pdata);

    return bk_size(pevent);
}

/*-- Periodic event ------------------------------------------------*/

INT read_periodic_event(char *pevent, INT off)
{
   short *pdata; // Change the data type to short

   // Init bank structure
   bk_init32(pevent);

   // Create a bank named "CR00" and specify the data type as TID_SHORT
   bk_create(pevent, "CR00", TID_SHORT, (void **)&pdata);

    // Repeat the loop 5000 times
    for (int repeat = 0; repeat < 400; repeat++) {
        for (int i = 0; i < data.size(); i++) {
            *pdata++ = data[i];
        }
    }

   // Close the bank
   bk_close(pevent, pdata);

   return bk_size(pevent);
}