Back Midas Rome Roody Rootana
  Midas DAQ System  Not logged in ELOG logo
Entry  05 Sep 2024, Jack Carlton, Forum, Python frontend rate limitations? frontend.pyfrontend.cxx
    Reply  05 Sep 2024, Ben Smith, Forum, Python frontend rate limitations? 
       Reply  05 Sep 2024, Stefan Ritt, Forum, Python frontend rate limitations? 
          Reply  06 Sep 2024, Jack Carlton, Forum, Python frontend rate limitations? 
          Reply  11 Sep 2024, Konstantin Olchanski, Forum, Python frontend rate limitations? 
       Reply  27 Sep 2024, Ben Smith, Forum, Python frontend rate limitations? 
    Reply  11 Sep 2024, Konstantin Olchanski, Forum, Python frontend rate limitations? 
       Reply  11 Sep 2024, Konstantin Olchanski, Forum, Python frontend rate limitations? 
Message ID: 2825     Entry time: 05 Sep 2024     Reply to this: 2826   2831
Author: Jack Carlton 
Topic: Forum 
Subject: Python frontend rate limitations? 
I'm trying to get a sense of the rate limitations of a python frontend. I 
understand this will vary from system to system.

I adapted two frontends from the example templates, one in C++ and one in python. 
Both simply fill a midas bank with a fixed length array of zeros at a given polled 
rate. However, the C++ frontend is about 100 times faster in both data and event 
rates. This seems slow, even for an interpreted language like python. Furthermore, 
I can effectively increase the maximum rate by concurrently running a second 
python frontend (this is not the case for the C++ frontend). In short, there is 
some limitation with using python here unrelated to hardware.

In my case, poll_func appears to be called at 100Hz at best. What limits the rate 
that poll_func is called in a python frontend? Is there a more appropriate 
solution for increasing the python frontend data/event rate than simply launching 
more frontends?

I've attached my C++ and python frontend files for reference.

Thanks,
Jack
Attachment 1: frontend.py  4 kB  Uploaded 05 Sep 2024  | Show | Hide all | Show all
Attachment 2: frontend.cxx  7 kB  Uploaded 05 Sep 2024  | Hide | Hide all | Show all
#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <string.h>
#include <iostream>
#include <fstream>
#include <sstream>
#include <vector>
#include "midas.h"
#include "mfe.h"
#include <stdlib.h> // Include the header for rand()
#include <random> // Include for random number generation


void trigger_update(INT, INT, void*);

/*-- Globals -------------------------------------------------------*/

/* The frontend name (client name) as seen by other MIDAS clients   */
const char *frontend_name = "DataSimulator";
/* The frontend file name, don't change it */
const char *frontend_file_name = __FILE__;

/* frontend_loop is called periodically if this variable is TRUE    */
BOOL frontend_call_loop = FALSE;

/* a frontend status page is displayed with this frequency in ms */
INT display_period = 1000;

/* maximum event size produced by this frontend */
INT max_event_size = 1024 * 1014;

/* maximum event size for fragmented events (EQ_FRAGMENTED) */
INT max_event_size_frag = 5 * max_event_size;

/* buffer size to hold events */
INT event_buffer_size = 5 * max_event_size;

// Define a vector to store 16-bit words
std::vector<int16_t> data; // Define a global vector to store 16-bit signed integers

// Global variable to keep track of the last poll time
std::chrono::steady_clock::time_point last_poll_time;
const std::chrono::microseconds polling_interval(300); // Poll every 300 microsecond

// Random number generator for generating data
std::mt19937 generator;
std::uniform_int_distribution<short> distribution(-32768, 32767); // Define the range of random values (short range)

// Global variable to hold the zero buffer
std::vector<short> zero_buffer;


/*-- Function declarations -----------------------------------------*/

INT frontend_init(void);
INT frontend_exit(void);
INT begin_of_run(INT run_number, char *error);
INT end_of_run(INT run_number, char *error);
INT pause_run(INT run_number, char *error);
INT resume_run(INT run_number, char *error);
INT frontend_loop(void);

INT read_trigger_event(char *pevent, INT off);
INT read_periodic_event(char *pevent, INT off);

INT poll_event(INT source, INT count, BOOL test);
INT interrupt_configure(INT cmd, INT source, POINTER_T adr);

/*-- Equipment list ------------------------------------------------*/

BOOL equipment_common_overwrite = TRUE;

EQUIPMENT equipment[] = {
   {"Data Simulator",              /* equipment name */
      {2, 0,                 /* event ID, trigger mask */
         "SYSTEM",           /* event buffer */
         EQ_POLLED,        /* equipment type */
         0,                  /* event source */
         "MIDAS",            /* format */
         TRUE,               /* enabled */
         RO_RUNNING | RO_TRANSITIONS |   /* read when running and on transitions */
         RO_ODB,             /* and update ODB */
         10,               /* read every sec */
         0,                  /* stop run after this event limit */
         0,                  /* number of sub events */
         TRUE,               /* log history */
         "", "", "",},
      read_trigger_event   /* readout routine */
   },

   {""}
};

/*-- Trigger Update ------------------------------------------------*/

void trigger_update(INT hDB, INT hkey,void*)
{

}


/*-- Frontend Init -------------------------------------------------*/

int frontend_init() {
    // Open the file for reading
    std::ifstream inputFile("fake_data.txt");

    if (!inputFile) {
        std::cerr << "Error opening the file." << std::endl;
        return 1;
    }

    std::cout << "Reading and converting data:" << std::endl;

    std::string line;
    while (std::getline(inputFile, line)) {
        std::istringstream iss(line);
        std::string token;

        while (std::getline(iss, token, ',')) {
            int16_t value;
            std::istringstream(token) >> value;
            data.push_back(value);
        }
    }

    // Print the converted data
    for (int i = 0; i < data.size(); i++) {
        std::cout << " " << data[i];
    }

    // Close the file
    inputFile.close();

    if (data.empty()) {
        std::cerr << "No data was converted." << std::endl;
    } else {
        std::cout << std::endl << "Conversion completed." << std::endl;
    }

    // Initialize random number generator
    std::random_device rd; // Obtain a random number from hardware
    generator = std::mt19937(rd()); // Seed the generator

    // Define the total number of zero data points
    const int total_data_size = 50000; // Adjust size as needed

    // Create and initialize the buffer of zeros
    zero_buffer.resize(total_data_size, 0);

    return SUCCESS;
}




/*-- Frontend Exit -------------------------------------------------*/

INT frontend_exit()
{
   return SUCCESS;
}

/*-- Begin of Run --------------------------------------------------*/

INT begin_of_run(INT run_number, char *error)
{
   return SUCCESS;
}

/*-- End of Run ----------------------------------------------------*/

INT end_of_run(INT run_number, char *error)
{
   return SUCCESS;
}

/*-- Pause Run -----------------------------------------------------*/

INT pause_run(INT run_number, char *error)
{
   return SUCCESS;
}

/*-- Resume Run ----------------------------------------------------*/

INT resume_run(INT run_number, char *error)
{
   return SUCCESS;
}

/*-- Frontend Loop -------------------------------------------------*/

INT frontend_loop()
{
   /* if frontend_call_loop is true, this routine gets called when
      the frontend is idle or once between every event */
   return SUCCESS;
}

/*------------------------------------------------------------------*/

/********************************************************************\

  Readout routines for different events

\********************************************************************/

/*-- Trigger event routines ----------------------------------------*/

INT poll_event(INT source, INT count, BOOL test) {
    // Get the current time
    auto now = std::chrono::steady_clock::now();
    
    // Check if enough time has passed since the last poll
    if (now - last_poll_time >= polling_interval) {
        // Update the last poll time
        last_poll_time = now;
        
        // Return TRUE to indicate that an event is available
        return TRUE;
    }
    
    // If test is TRUE, don't return anything
    if (test) {
        return FALSE;
    }
    
    // Otherwise, return FALSE to indicate no event available
    return FALSE;
}

/*-- Interrupt configuration ---------------------------------------*/

INT interrupt_configure(INT cmd, INT source, POINTER_T adr)
{
   switch (cmd) {
   case CMD_INTERRUPT_ENABLE:
      break;
   case CMD_INTERRUPT_DISABLE:
      break;
   case CMD_INTERRUPT_ATTACH:
      break;
   case CMD_INTERRUPT_DETACH:
      break;
   }
   return SUCCESS;
}

/*-- Event readout -------------------------------------------------*/

INT read_trigger_event(char *pevent, INT off)
{
    short *pdata;

    // Init bank structure
    bk_init32(pevent);

    // Create a bank named "CR00" and specify the data type as TID_SHORT
    bk_create(pevent, "CR00", TID_SHORT, (void **)&pdata);

    // Use memcpy to copy the buffer of zeros into the MIDAS bank
    memcpy(pdata, zero_buffer.data(), zero_buffer.size() * sizeof(short));

    // Adjust pdata pointer
    pdata += zero_buffer.size();  // Move the pointer past the copied data

    // Close the bank
    bk_close(pevent, pdata);

    return bk_size(pevent);
}

/*-- Periodic event ------------------------------------------------*/

INT read_periodic_event(char *pevent, INT off)
{
   short *pdata; // Change the data type to short

   // Init bank structure
   bk_init32(pevent);

   // Create a bank named "CR00" and specify the data type as TID_SHORT
   bk_create(pevent, "CR00", TID_SHORT, (void **)&pdata);

    // Repeat the loop 5000 times
    for (int repeat = 0; repeat < 400; repeat++) {
        for (int i = 0; i < data.size(); i++) {
            *pdata++ = data[i];
        }
    }

   // Close the bank
   bk_close(pevent, pdata);

   return bk_size(pevent);
}
ELOG V3.1.4-2e1708b5