HsFileSchema Struct Reference

Inheritance diagram for HsFileSchema:

Collaboration diagram for HsFileSchema:

Public Member Functions
	HsFileSchema ()
	~HsFileSchema ()
void	print (bool print_tags=true) const
int	flush_buffers ()
int	close ()
int	write_event (const time_t t, const char *data, const int data_size)
int	read_last_written (const time_t timestamp, const int debug, time_t *last_written)
int	read_data (const time_t start_time, const time_t end_time, const int num_var, const std::vector< int > &var_schema_index, const int var_index[], const int debug, std::vector< time_t > &last_time, MidasHistoryBufferInterface *buffer[])
Public Member Functions inherited from HsSchema
	HsSchema ()
virtual	~HsSchema ()
virtual int	match_event_var (const char *event_name, const char *var_name, const int var_index)

Public Attributes
std::string	file_name
int	record_size = 0
int	data_offset = 0
int	last_size = 0
int	writer_fd = -1
int	record_buffer_size = 0
char *	record_buffer = NULL
Public Attributes inherited from HsSchema
std::string	event_name
time_t	time_from = 0
time_t	time_to = 0
std::vector< HsSchemaEntry >	variables
std::vector< int >	offsets
int	n_bytes = 0
int	count_write_undersize = 0
int	count_write_oversize = 0
int	write_max_size = 0
int	write_min_size = 0
bool	disabled = true

Detailed Description

Definition at line 741 of file history_schema.cxx.

Constructor & Destructor Documentation

HsFileSchema()

HsFileSchema::HsFileSchema ( )

inline

Definition at line 750 of file history_schema.cxx.

   {
      // empty
   }

~HsFileSchema()

HsFileSchema::~HsFileSchema ( )

inline

Definition at line 755 of file history_schema.cxx.

   {
      close();
      record_size = 0;
      data_offset = 0;
      last_size = 0;
      writer_fd = -1;
      if (record_buffer) {
         free(record_buffer);
         record_buffer = NULL;
      }
      record_buffer_size = 0;
   }

Here is the call graph for this function:

Member Function Documentation

close()

int HsFileSchema::close ( )

virtual

Implements HsSchema.

Definition at line 2550 of file history_schema.cxx.

{
   if (writer_fd >= 0) {
      ::close(writer_fd);
      writer_fd = -1;
   }
   return HS_SUCCESS;
}

Here is the call graph for this function:

Here is the caller graph for this function:

flush_buffers()

int HsFileSchema::flush_buffers ( )

inlinevirtual

Implements HsSchema.

Definition at line 770 of file history_schema.cxx.

{ return HS_SUCCESS; };

print()

void HsFileSchema::print ( bool  print_tags = true ) const

virtual

Reimplemented from HsSchema.

Definition at line 811 of file history_schema.cxx.

{
   unsigned nv = this->variables.size();
   printf("event [%s], file_name [%s], time %s..%s, %d variables, %d bytes, dat_offset %d, record_size %d\n", this->event_name.c_str(), this->file_name.c_str(), TimeToString(this->time_from).c_str(), TimeToString(this->time_to).c_str(), nv, n_bytes, data_offset, record_size);
   if (print_tags) {
      for (unsigned j=0; j<nv; j++)
         printf("  %d: name [%s], type [%s] tid %d, n_data %d, n_bytes %d, offset %d\n", j, this->variables[j].name.c_str(), rpc_tid_name(this->variables[j].type), this->variables[j].type, this->variables[j].n_data, this->variables[j].n_bytes, this->offsets[j]);
   }
}

Here is the call graph for this function:

Here is the caller graph for this function:

read_data()

int HsFileSchema::read_data ( const time_t  start_time, const time_t  end_time, const int  num_var, const std::vector< int > &  var_schema_index, const int  var_index[], const int  debug, std::vector< time_t > &  last_time, MidasHistoryBufferInterface *  buffer[]  )

virtual

Implements HsSchema.

Definition at line 2801 of file history_schema.cxx.

{
   HsFileSchema* s = this;
 
   if (debug)
      printf("FileHistory::read_data: file %s, schema time %s..%s, read time %s..%s, %d vars\n", s->file_name.c_str(), TimeToString(s->time_from).c_str(), TimeToString(s->time_to).c_str(), TimeToString(start_time).c_str(), TimeToString(end_time).c_str(), num_var);
 
   //if (1) {
   //   printf("Last time: ");
   //   for (int i=0; i<num_var; i++) {
   //      printf(" %s", TimeToString(last_time[i]).c_str());
   //   }
   //   printf("\n");
   //}
 
   if (debug) {
      printf("FileHistory::read_data: file %s map", s->file_name.c_str());
      for (size_t i=0; i<var_schema_index.size(); i++) {
         printf(" %2d", var_schema_index[i]);
      }
      printf("\n");
   }
 
   int fd = ::open(s->file_name.c_str(), O_RDONLY);
   if (fd < 0) {
      cm_msg(MERROR, "FileHistory::read_data", "Cannot read \'%s\', open() errno %d (%s)", s->file_name.c_str(), errno, strerror(errno));
      return HS_FILE_ERROR;
   }
 
   off_t file_size = ::lseek(fd, 0, SEEK_END);
 
   if (file_size == (off_t)-1) {
      cm_msg(MERROR, "FileHistory::read_data", "Cannot read \'%s\', fseek(SEEK_END) errno %d (%s)", s->file_name.c_str(), errno, strerror(errno));
      ::close(fd);
      return HS_FILE_ERROR;
   }
 
   int nrec = (file_size - s->data_offset)/s->record_size;
   if (nrec < 0)
      nrec = 0;
 
   if (nrec < 1) {
      ::close(fd);
      return HS_SUCCESS;
   }
 
   int iunused = 0;
   time_t tunused = 0;
   int irec = 0;
   time_t trec   = 0;
   time_t tstart = 0;
   time_t tend   = 0;
 
   int status = FindTime(s->file_name.c_str(), fd, s->data_offset, s->record_size, nrec, start_time, &iunused, &tunused, &irec, &trec, &tstart, &tend, 0*debug);
 
   if (status != HS_SUCCESS) {
      ::close(fd);
      return HS_FILE_ERROR;
   }
 
   if (debug) {
      printf("FindTime %d, nrec %d, (%d, %s) (%d, %s), tstart %s, tend %s, want %s\n", status, nrec, iunused, TimeToString(tunused).c_str(), irec, TimeToString(trec).c_str(), TimeToString(tstart).c_str(), TimeToString(tend).c_str(), TimeToString(start_time).c_str());
   }
 
   if (irec < 0 || irec >= nrec) {
      // all data in this file is older than start_time
 
      ::close(fd);
 
      if (debug)
         printf("FileHistory::read: file %s, schema time %s..%s, read time %s..%s, file time %s..%s, data in this file is too old\n", s->file_name.c_str(), TimeToString(s->time_from).c_str(), TimeToString(s->time_to).c_str(), TimeToString(start_time).c_str(), TimeToString(end_time).c_str(), TimeToString(tstart).c_str(), TimeToString(tend).c_str());
 
      return HS_SUCCESS;
   }
 
   if (tstart < s->time_from) {
      // data starts before time declared in schema
 
      ::close(fd);
 
      cm_msg(MERROR, "FileHistory::read_data", "Bad history file \'%s\': timestamp of first data %s is before schema start time %s", s->file_name.c_str(), TimeToString(tstart).c_str(), TimeToString(s->time_from).c_str());
 
      return HS_FILE_ERROR;
   }
 
   if (tend && s->time_to && tend > s->time_to) {
      // data ends after time declared in schema (overlaps with next file)
 
      ::close(fd);
 
      cm_msg(MERROR, "FileHistory::read_data", "Bad history file \'%s\': timestamp of last data %s is after schema end time %s", s->file_name.c_str(), TimeToString(tend).c_str(), TimeToString(s->time_to).c_str());
 
      return HS_FILE_ERROR;
   }
 
   for (int i=0; i<num_var; i++) {
      int si = var_schema_index[i];
      if (si < 0)
         continue;
         
      if (trec < last_time[i]) { // protect against duplicate and non-monotonous data
         ::close(fd);
 
         cm_msg(MERROR, "FileHistory::read_data", "Internal history error at file \'%s\': variable %d data timestamp %s is before last timestamp %s", s->file_name.c_str(), i, TimeToString(trec).c_str(), TimeToString(last_time[i]).c_str());
         
         return HS_FILE_ERROR;
      }
   }
 
   int count = 0;
 
   off_t fpos = s->data_offset + irec*s->record_size;
   
   off_t xpos = ::lseek(fd, fpos, SEEK_SET);
   if (xpos == (off_t)-1) {
      cm_msg(MERROR, "FileHistory::read_data", "Cannot read \'%s\', lseek(%zu) errno %d (%s)", s->file_name.c_str(), (size_t)fpos, errno, strerror(errno));
      ::close(fd);
      return HS_FILE_ERROR;
   }
   
   char* buf = new char[s->record_size];
 
   int prec = irec;
 
   while (1) {
      status = ::read(fd, buf, s->record_size);
      if (status == 0) // EOF
         break;
      if (status == -1) {
         cm_msg(MERROR, "FileHistory::read_data", "Cannot read \'%s\', read() errno %d (%s)", s->file_name.c_str(), errno, strerror(errno));
         break;
      }
      if (status != s->record_size) {
         cm_msg(MERROR, "FileHistory::read_data", "Cannot read \'%s\', short read() returned %d instead of %d bytes", s->file_name.c_str(), status, s->record_size);
         break;
      }
 
      prec++;
 
      bool past_end_of_last_file = (s->time_to == 0) && (prec > nrec);
 
      time_t t = *(DWORD*)buf;
      
      if (debug > 1)
         printf("FileHistory::read: file %s, schema time %s..%s, read time %s..%s, row time %s\n", s->file_name.c_str(), TimeToString(s->time_from).c_str(), TimeToString(s->time_to).c_str(), TimeToString(start_time).c_str(), TimeToString(end_time).c_str(), TimeToString(t).c_str());
      
      if (t < trec) {
         delete[] buf;
         ::close(fd);
         cm_msg(MERROR, "FileHistory::read_data", "Bad history file \'%s\': record %d timestamp %s is before start time %s", s->file_name.c_str(), irec + count, TimeToString(t).c_str(), TimeToString(trec).c_str());
         return HS_FILE_ERROR;
      }
      
      if (tend && (t > tend) && !past_end_of_last_file) {
         delete[] buf;
         ::close(fd);
         cm_msg(MERROR, "FileHistory::read_data", "Bad history file \'%s\': record %d timestamp %s is after last timestamp %s", s->file_name.c_str(), irec + count, TimeToString(t).c_str(), TimeToString(tend).c_str());
         return HS_FILE_ERROR;
      }
      
      if (t > end_time)
         break;
      
      char* data = buf + 4;
      
      for (int i=0; i<num_var; i++) {
         int si = var_schema_index[i];
         if (si < 0)
            continue;
         
         if (t < last_time[i]) { // protect against duplicate and non-monotonous data
            delete[] buf;
            ::close(fd);
 
            cm_msg(MERROR, "FileHistory::read_data", "Bad history file \'%s\': record %d timestamp %s is before timestamp %s of variable %d", s->file_name.c_str(), irec + count, TimeToString(t).c_str(), TimeToString(last_time[i]).c_str(), i);
 
            return HS_FILE_ERROR;
         }
         
         double v = 0;
         void* ptr = data + s->offsets[si];
         
         int ii = var_index[i];
         assert(ii >= 0);
         assert(ii < s->variables[si].n_data);
         
         switch (s->variables[si].type) {
         default:
            // unknown data type
            v = 0;
            break;
         case TID_BYTE:
            v = ((unsigned char*)ptr)[ii];
            break;
         case TID_SBYTE:
            v = ((signed char *)ptr)[ii];
            break;
         case TID_CHAR:
            v = ((char*)ptr)[ii];
            break;
         case TID_WORD:
            v = ((unsigned short *)ptr)[ii];
            break;
         case TID_SHORT:
            v = ((signed short *)ptr)[ii];
            break;
         case TID_DWORD:
            v = ((unsigned int *)ptr)[ii];
            break;
         case TID_INT:
            v = ((int *)ptr)[ii];
            break;
         case TID_BOOL:
            v = ((unsigned int *)ptr)[ii];
            break;
         case TID_FLOAT:
            v = ((float*)ptr)[ii];
            break;
         case TID_DOUBLE:
            v = ((double*)ptr)[ii];
            break;
         }
         
         buffer[i]->Add(t, v);
         last_time[i] = t;
      }
      count++;
   }
 
   delete[] buf;
 
   ::close(fd);
 
   if (debug) {
      printf("FileHistory::read_data: file %s map", s->file_name.c_str());
      for (size_t i=0; i<var_schema_index.size(); i++) {
         printf(" %2d", var_schema_index[i]);
      }
      printf(" read %d rows\n", count);
   }
 
   if (debug)
      printf("FileHistory::read: file %s, schema time %s..%s, read time %s..%s, %d vars, read %d rows\n", s->file_name.c_str(), TimeToString(s->time_from).c_str(), TimeToString(s->time_to).c_str(), TimeToString(start_time).c_str(), TimeToString(end_time).c_str(), num_var, count);
 
   return HS_SUCCESS;
}

Here is the call graph for this function:

read_last_written()

int HsFileSchema::read_last_written ( const time_t  timestamp, const int  debug, time_t *  last_written  )

virtual

Implements HsSchema.

Definition at line 2721 of file history_schema.cxx.

{
   int status;
   HsFileSchema* s = this;
 
   if (debug)
      printf("FileHistory::read_last_written: file %s, schema time %s..%s, timestamp %s\n", s->file_name.c_str(), TimeToString(s->time_from).c_str(), TimeToString(s->time_to).c_str(), TimeToString(timestamp).c_str());
 
   int fd = open(s->file_name.c_str(), O_RDONLY);
   if (fd < 0) {
      cm_msg(MERROR, "FileHistory::read_last_written", "Cannot read \'%s\', open() errno %d (%s)", s->file_name.c_str(), errno, strerror(errno));
      return HS_FILE_ERROR;
   }
 
   int file_size = ::lseek(fd, 0, SEEK_END);
 
   int nrec = (file_size - s->data_offset)/s->record_size;
   if (nrec < 0)
      nrec = 0;
 
   if (nrec < 1) {
      ::close(fd);
      if (last_written)
         *last_written = 0;
      return HS_SUCCESS;
   }
 
   time_t lw = 0;
 
   // read last record to check if desired time is inside or outside of the file
 
   if (1) {
      char* buf = new char[s->record_size];
 
      status = ReadRecord(s->file_name.c_str(), fd, s->data_offset, s->record_size, nrec - 1, buf);
      if (status != HS_SUCCESS) {
         delete[] buf;
         ::close(fd);
         return HS_FILE_ERROR;
      }
 
      lw = *(DWORD*)buf;
 
      delete[] buf;
   }
 
   if (lw >= timestamp) {
      int irec = 0;
      time_t trec = 0;
      int iunused = 0;
      time_t tunused = 0;
      time_t tstart = 0; // not used
      time_t tend   = 0; // not used
 
      status = FindTime(s->file_name.c_str(), fd, s->data_offset, s->record_size, nrec, timestamp, &irec, &trec, &iunused, &tunused, &tstart, &tend, 0*debug);
      if (status != HS_SUCCESS) {
         ::close(fd);
         return HS_FILE_ERROR;
      }
 
      assert(trec < timestamp);
 
      lw = trec;
   }
 
   if (last_written)
      *last_written = lw;
 
   if (debug)
      printf("FileHistory::read_last_written: file %s, schema time %s..%s, timestamp %s, last_written %s\n", s->file_name.c_str(), TimeToString(s->time_from).c_str(), TimeToString(s->time_to).c_str(), TimeToString(timestamp).c_str(), TimeToString(lw).c_str());
 
   assert(lw < timestamp);
 
   ::close(fd);
 
   return HS_SUCCESS;
}

Here is the call graph for this function:

write_event()

int HsFileSchema::write_event ( const time_t  t, const char *  data, const int  data_size  )

virtual

Implements HsSchema.

Definition at line 2456 of file history_schema.cxx.

{
   HsFileSchema* s = this;
 
   int status;
 
   if (s->writer_fd < 0) {
      s->writer_fd = open(s->file_name.c_str(), O_RDWR);
      if (s->writer_fd < 0) {
         cm_msg(MERROR, "FileHistory::write_event", "Cannot write to \'%s\', open() errno %d (%s)", s->file_name.c_str(), errno, strerror(errno));
         return HS_FILE_ERROR;
      }
 
      int file_size = lseek(s->writer_fd, 0, SEEK_END);
 
      int nrec = (file_size - s->data_offset)/s->record_size;
      if (nrec < 0)
         nrec = 0;
      int data_end = s->data_offset + nrec*s->record_size;
 
      //printf("file_size %d, nrec %d, data_end %d\n", file_size, nrec, data_end);
 
      if (data_end != file_size) {
         if (nrec > 0)
            cm_msg(MERROR, "FileHistory::write_event", "File \'%s\' may be truncated, data offset %d, record size %d, file size: %d, should be %d, truncating the file", s->file_name.c_str(), s->data_offset, s->record_size, file_size, data_end);
 
         status = lseek(s->writer_fd, data_end, SEEK_SET);
         if (status < 0) {
            cm_msg(MERROR, "FileHistory::write_event", "Cannot seek \'%s\' to offset %d, lseek() errno %d (%s)", s->file_name.c_str(), data_end, errno, strerror(errno));
            return HS_FILE_ERROR;
         }
         status = ftruncate(s->writer_fd, data_end);
         if (status < 0) {
            cm_msg(MERROR, "FileHistory::write_event", "Cannot truncate \'%s\' to size %d, ftruncate() errno %d (%s)", s->file_name.c_str(), data_end, errno, strerror(errno));
            return HS_FILE_ERROR;
         }
      }
   }
 
   int expected_size = s->record_size - 4;
 
   // sanity check: record_size and n_bytes are computed from the byte counts in the file header
   assert(expected_size == s->n_bytes);
 
   if (s->last_size == 0)
      s->last_size = expected_size;
 
   if (data_size != s->last_size) {
      cm_msg(MERROR, "FileHistory::write_event", "Event \'%s\' data size mismatch, expected %d bytes, got %d bytes, previously %d bytes", s->event_name.c_str(), expected_size, data_size, s->last_size);
      //printf("schema:\n");
      //s->print();
 
      if (data_size < expected_size)
         return HS_FILE_ERROR;
 
      // truncate for now
      // data_size = expected_size;
      s->last_size = data_size;
   }
 
   int size = 4 + expected_size;
 
   if (size != s->record_buffer_size) {
      s->record_buffer = (char*)realloc(s->record_buffer, size);
      assert(s->record_buffer != NULL);
      s->record_buffer_size = size;
   }
 
   memcpy(s->record_buffer, &t, 4);
   memcpy(s->record_buffer+4, data, expected_size);
 
   status = write(s->writer_fd, s->record_buffer, size);
   if (status != size) {
      cm_msg(MERROR, "FileHistory::write_event", "Cannot write to \'%s\', write(%d) returned %d, errno %d (%s)", s->file_name.c_str(), size, status, errno, strerror(errno));
      return HS_FILE_ERROR;
   }
 
#if 0
   status = write(s->writer_fd, &t, 4);
   if (status != 4) {
      cm_msg(MERROR, "FileHistory::write_event", "Cannot write to \'%s\', write(timestamp) errno %d (%s)", s->file_name.c_str(), errno, strerror(errno));
      return HS_FILE_ERROR;
   }
 
   status = write(s->writer_fd, data, expected_size);
   if (status != expected_size) {
      cm_msg(MERROR, "FileHistory::write_event", "Cannot write to \'%s\', write(%d) errno %d (%s)", s->file_name.c_str(), data_size, errno, strerror(errno));
      return HS_FILE_ERROR;
   }
#endif
 
   return HS_SUCCESS;
}

Here is the call graph for this function:

Member Data Documentation

data_offset

int HsFileSchema::data_offset = 0

Definition at line 744 of file history_schema.cxx.

file_name

std::string HsFileSchema::file_name

Definition at line 742 of file history_schema.cxx.

last_size

int HsFileSchema::last_size = 0

Definition at line 745 of file history_schema.cxx.

record_buffer

char* HsFileSchema::record_buffer = NULL

Definition at line 748 of file history_schema.cxx.

record_buffer_size

int HsFileSchema::record_buffer_size = 0

Definition at line 747 of file history_schema.cxx.

record_size

int HsFileSchema::record_size = 0

Definition at line 743 of file history_schema.cxx.

writer_fd

int HsFileSchema::writer_fd = -1

Definition at line 746 of file history_schema.cxx.

---

The documentation for this struct was generated from the following file:

• src/history_schema.cxx