Line data Source code
1 : /********************************************************************\
2 :
3 : Name: history_schema.cxx
4 : Created by: Konstantin Olchanski
5 :
6 : Contents: Schema based MIDAS history. Available drivers:
7 : FileHistory: storage of data in binary files (replacement for the traditional MIDAS history)
8 : MysqlHistory: storage of data in MySQL database (replacement for the ODBC based SQL history)
9 : PgsqlHistory: storage of data in PostgreSQL database
10 : SqliteHistory: storage of data in SQLITE3 database (not suitable for production use)
11 :
12 : \********************************************************************/
13 :
14 : #undef NDEBUG // midas required assert() to be always enabled
15 :
16 : #include "midas.h"
17 : #include "msystem.h"
18 : #include "mstrlcpy.h"
19 :
20 : #include <math.h>
21 :
22 : #include <vector>
23 : #include <list>
24 : #include <string>
25 : #include <map>
26 : #include <algorithm>
27 :
28 : // make mysql/my_global.h happy - it redefines closesocket()
29 : #undef closesocket
30 :
31 : //
32 : // benchmarks
33 : //
34 : // /usr/bin/time ./linux/bin/mh2sql . /ladd/iris_data2/alpha/alphacpc09-elog-history/history/121019.hst
35 : // -rw-r--r-- 1 alpha users 161028048 Oct 19 2012 /ladd/iris_data2/alpha/alphacpc09-elog-history/history/121019.hst
36 : // flush 10000, sync=OFF -> 51.51user 1.51system 0:53.76elapsed 98%CPU
37 : // flush 1000000, sync=NORMAL -> 51.83user 2.09system 1:08.37elapsed 78%CPU (flush never activated)
38 : // flush 100000, sync=NORMAL -> 51.38user 1.94system 1:06.94elapsed 79%CPU
39 : // flush 10000, sync=NORMAL -> 51.37user 2.03system 1:31.63elapsed 58%CPU
40 : // flush 1000, sync=NORMAL -> 52.16user 2.70system 4:38.58elapsed 19%CPU
41 :
42 : ////////////////////////////////////////
43 : // MIDAS includes //
44 : ////////////////////////////////////////
45 :
46 : #include "midas.h"
47 : #include "history.h"
48 :
49 : ////////////////////////////////////////
50 : // helper stuff //
51 : ////////////////////////////////////////
52 :
53 : #define FREE(x) { if (x) free(x); (x) = NULL; }
54 :
55 0 : static char* skip_spaces(char* s)
56 : {
57 0 : while (*s) {
58 0 : if (!isspace(*s))
59 0 : break;
60 0 : s++;
61 : }
62 0 : return s;
63 : }
64 :
65 0 : static std::string TimeToString(time_t t)
66 : {
67 0 : const char* sign = "";
68 :
69 0 : if (t == 0)
70 0 : return "0";
71 :
72 0 : time_t tt = t;
73 :
74 0 : if (t < 0) {
75 0 : sign = "-";
76 0 : tt = -t;
77 : }
78 :
79 0 : assert(tt > 0);
80 :
81 0 : std::string v;
82 0 : while (tt) {
83 0 : char c = '0' + (char)(tt%10);
84 0 : tt /= 10;
85 0 : v = c + v;
86 : }
87 :
88 0 : v = sign + v;
89 :
90 : //printf("time %.0f -> %s\n", (double)t, v.c_str());
91 :
92 0 : return v;
93 0 : }
94 :
95 0 : static std::string SmallIntToString(int i)
96 : {
97 : //int ii = i;
98 :
99 0 : if (i == 0)
100 0 : return "0";
101 :
102 0 : assert(i > 0);
103 :
104 0 : std::string v;
105 0 : while (i) {
106 0 : char c = '0' + (char)(i%10);
107 0 : i /= 10;
108 0 : v = c + v;
109 : }
110 :
111 : //printf("SmallIntToString: %d -> %s\n", ii, v.c_str());
112 :
113 0 : return v;
114 0 : }
115 :
116 0 : static bool MatchEventName(const char* event_name, const char* var_event_name)
117 : {
118 : // new-style event name: "equipment_name/variable_name:tag_name"
119 : // old-style event name: "equipment_name:tag_name" ("variable_name" is missing)
120 0 : bool newStyleEventName = (strchr(var_event_name, '/')!=NULL);
121 :
122 : //printf("looking for event_name [%s], try table [%s] event name [%s], new style [%d]\n", var_event_name, table_name, event_name, newStyleEventName);
123 :
124 0 : if (strcasecmp(event_name, var_event_name) == 0) {
125 0 : return true;
126 0 : } else if (newStyleEventName) {
127 0 : return false;
128 : } else { // for old style names, need more parsing
129 0 : bool match = false;
130 :
131 0 : const char* s = event_name;
132 0 : for (int j=0; s[j]; j++) {
133 :
134 0 : if ((var_event_name[j]==0) && (s[j]=='/')) {
135 0 : match = true;
136 0 : break;
137 : }
138 :
139 0 : if ((var_event_name[j]==0) && (s[j]=='_')) {
140 0 : match = true;
141 0 : break;
142 : }
143 :
144 0 : if (var_event_name[j]==0) {
145 0 : match = false;
146 0 : break;
147 : }
148 :
149 0 : if (tolower(var_event_name[j]) != tolower(s[j])) { // does not work for UTF-8 Unicode
150 0 : match = false;
151 0 : break;
152 : }
153 : }
154 :
155 0 : return match;
156 : }
157 : }
158 :
159 0 : static bool MatchTagName(const char* tag_name, int n_data, const char* var_tag_name, const int var_tag_index)
160 : {
161 : char alt_tag_name[1024]; // maybe this is an array without "Names"?
162 0 : sprintf(alt_tag_name, "%s[%d]", var_tag_name, var_tag_index);
163 :
164 : //printf(" looking for tag [%s] alt [%s], try column name [%s]\n", var_tag_name, alt_tag_name, tag_name);
165 :
166 0 : if (strcasecmp(tag_name, var_tag_name) == 0)
167 0 : if (var_tag_index >= 0 && var_tag_index < n_data)
168 0 : return true;
169 :
170 0 : if (strcasecmp(tag_name, alt_tag_name) == 0)
171 0 : return true;
172 :
173 0 : return false;
174 : }
175 :
176 0 : static void PrintTags(int ntags, const TAG tags[])
177 : {
178 0 : for (int i=0; i<ntags; i++)
179 0 : printf("tag %d: %s %s[%d]\n", i, rpc_tid_name(tags[i].type), tags[i].name, tags[i].n_data);
180 0 : }
181 :
182 : // convert MIDAS event name to something acceptable as an SQL identifier - table name, column name, etc
183 :
184 0 : static std::string MidasNameToSqlName(const char* s)
185 : {
186 0 : std::string out;
187 :
188 0 : for (int i=0; s[i]!=0; i++) {
189 0 : char c = s[i];
190 0 : if (isalpha(c) || isdigit(c))
191 0 : out += tolower(c); // does not work for UTF-8 Unicode
192 : else
193 0 : out += '_';
194 : }
195 :
196 0 : return out;
197 0 : }
198 :
199 : // convert MIDAS event name to something acceptable as a file name
200 :
201 0 : static std::string MidasNameToFileName(const char* s)
202 : {
203 0 : std::string out;
204 :
205 0 : for (int i=0; s[i]!=0; i++) {
206 0 : char c = s[i];
207 0 : if (isalpha(c) || isdigit(c))
208 0 : out += tolower(c); // does not work for UTF-8 Unicode
209 : else
210 0 : out += '_';
211 : }
212 :
213 0 : return out;
214 0 : }
215 :
216 : // compare event names
217 :
218 0 : static int event_name_cmp(const std::string& e1, const char* e2)
219 : {
220 0 : return strcasecmp(e1.c_str(), e2);
221 : }
222 :
223 : // compare variable names
224 :
225 0 : static int var_name_cmp(const std::string& v1, const char* v2)
226 : {
227 0 : return strcasecmp(v1.c_str(), v2);
228 : }
229 :
230 : ////////////////////////////////////////
231 : // SQL data types //
232 : ////////////////////////////////////////
233 :
234 : #ifdef HAVE_SQLITE
235 : static const char *sql_type_sqlite[TID_LAST] = {
236 : "xxxINVALIDxxxNULL", // TID_NULL
237 : "INTEGER", // TID_UINT8
238 : "INTEGER", // TID_INT8
239 : "TEXT", // TID_CHAR
240 : "INTEGER", // TID_UINT16
241 : "INTEGER", // TID_INT16
242 : "INTEGER", // TID_UINT32
243 : "INTEGER", // TID_INT32
244 : "INTEGER", // TID_BOOL
245 : "REAL", // TID_FLOAT
246 : "REAL", // TID_DOUBLE
247 : "INTEGER", // TID_BITFIELD
248 : "TEXT", // TID_STRING
249 : "xxxINVALIDxxxARRAY",
250 : "xxxINVALIDxxxSTRUCT",
251 : "xxxINVALIDxxxKEY",
252 : "xxxINVALIDxxxLINK"
253 : };
254 : #endif
255 :
256 : #ifdef HAVE_PGSQL
257 : static const char *sql_type_pgsql[TID_LAST] = {
258 : "xxxINVALIDxxxNULL", // TID_NULL
259 : "smallint", // TID_BYTE
260 : "smallint", // TID_SBYTE
261 : "char(1)", // TID_CHAR
262 : "integer", // TID_WORD
263 : "smallint", // TID_SHORT
264 : "bigint", // TID_DWORD
265 : "integer", // TID_INT
266 : "smallint", // TID_BOOL
267 : "real", // TID_FLOAT
268 : "double precision", // TID_DOUBLE
269 : "bigint", // TID_BITFIELD
270 : "text", // TID_STRING
271 : "xxxINVALIDxxxARRAY",
272 : "xxxINVALIDxxxSTRUCT",
273 : "xxxINVALIDxxxKEY",
274 : "xxxINVALIDxxxLINK"
275 : };
276 : #endif
277 :
278 : #ifdef HAVE_MYSQL
279 : static const char *sql_type_mysql[TID_LAST] = {
280 : "xxxINVALIDxxxNULL", // TID_NULL
281 : "tinyint unsigned", // TID_BYTE
282 : "tinyint", // TID_SBYTE
283 : "char", // TID_CHAR
284 : "smallint unsigned", // TID_WORD
285 : "smallint", // TID_SHORT
286 : "integer unsigned", // TID_DWORD
287 : "integer", // TID_INT
288 : "tinyint", // TID_BOOL
289 : "float", // TID_FLOAT
290 : "double", // TID_DOUBLE
291 : "integer unsigned", // TID_BITFIELD
292 : "VARCHAR", // TID_STRING
293 : "xxxINVALIDxxxARRAY",
294 : "xxxINVALIDxxxSTRUCT",
295 : "xxxINVALIDxxxKEY",
296 : "xxxINVALIDxxxLINK"
297 : };
298 : #endif
299 :
300 0 : void DoctorPgsqlColumnType(std::string* col_type, const char* index_type)
301 : {
302 0 : if (*col_type == index_type)
303 0 : return;
304 :
305 0 : if (*col_type == "bigint" && strcmp(index_type, "int8")==0) {
306 0 : *col_type = index_type;
307 0 : return;
308 : }
309 :
310 0 : if (*col_type == "integer" && strcmp(index_type, "int4")==0) {
311 0 : *col_type = index_type;
312 0 : return;
313 : }
314 :
315 0 : if (*col_type == "smallint" && strcmp(index_type, "int2")==0) {
316 0 : *col_type = index_type;
317 0 : return;
318 : }
319 :
320 0 : cm_msg(MERROR, "SqlHistory", "Cannot use this SQL database, incompatible column names: created column type [%s] is reported with column type [%s]", index_type, col_type->c_str());
321 0 : cm_msg_flush_buffer();
322 0 : abort();
323 : }
324 :
325 0 : void DoctorSqlColumnType(std::string* col_type, const char* index_type)
326 : {
327 0 : if (*col_type == index_type)
328 0 : return;
329 :
330 0 : if (*col_type == "int(10) unsigned" && strcmp(index_type, "integer unsigned")==0) {
331 0 : *col_type = index_type;
332 0 : return;
333 : }
334 :
335 0 : if (*col_type == "int(11)" && strcmp(index_type, "integer")==0) {
336 0 : *col_type = index_type;
337 0 : return;
338 : }
339 :
340 0 : if (*col_type == "integer" && strcmp(index_type, "int(11)")==0) {
341 0 : *col_type = index_type;
342 0 : return;
343 : }
344 :
345 : // MYSQL 8.0.23
346 :
347 0 : if (*col_type == "int" && strcmp(index_type, "integer")==0) {
348 0 : *col_type = index_type;
349 0 : return;
350 : }
351 :
352 0 : if (*col_type == "int unsigned" && strcmp(index_type, "integer unsigned")==0) {
353 0 : *col_type = index_type;
354 0 : return;
355 : }
356 :
357 0 : cm_msg(MERROR, "SqlHistory", "Cannot use this SQL database, incompatible column names: created column type [%s] is reported with column type [%s]", index_type, col_type->c_str());
358 0 : cm_msg_flush_buffer();
359 0 : abort();
360 : }
361 :
362 : #if 0
363 : static int sql2midasType_mysql(const char* name)
364 : {
365 : for (int tid=0; tid<TID_LAST; tid++)
366 : if (strcasecmp(name, sql_type_mysql[tid])==0)
367 : return tid;
368 : // FIXME!
369 : printf("sql2midasType: Cannot convert SQL data type \'%s\' to a MIDAS data type!\n", name);
370 : return 0;
371 : }
372 : #endif
373 :
374 : #if 0
375 : static int sql2midasType_sqlite(const char* name)
376 : {
377 : if (strcmp(name, "INTEGER") == 0)
378 : return TID_INT;
379 : if (strcmp(name, "REAL") == 0)
380 : return TID_DOUBLE;
381 : if (strcmp(name, "TEXT") == 0)
382 : return TID_STRING;
383 : // FIXME!
384 : printf("sql2midasType: Cannot convert SQL data type \'%s\' to a MIDAS data type!\n", name);
385 : return 0;
386 : }
387 : #endif
388 :
389 : ////////////////////////////////////////
390 : // Schema base classes //
391 : ////////////////////////////////////////
392 :
393 : struct HsSchemaEntry {
394 : std::string tag_name; // tag name from MIDAS
395 : std::string tag_type; // tag type from MIDAS
396 : std::string name; // entry name, same as tag_name except when read from SQL history when it could be the SQL column name
397 : int type = 0; // MIDAS data type TID_xxx
398 : int n_data = 0; // MIDAS array size
399 : int n_bytes = 0; // n_data * size of MIDAS data type (only used by HsFileSchema?)
400 : };
401 :
402 : class HsSchema
403 : {
404 : public:
405 :
406 : // event schema definitions
407 : std::string fEventName;
408 : time_t fTimeFrom = 0;
409 : time_t fTimeTo = 0;
410 : std::vector<HsSchemaEntry> fVariables;
411 : std::vector<int> fOffsets;
412 : size_t fNumBytes = 0;
413 :
414 : // run time data used by hs_write_event()
415 : int fCountWriteUndersize = 0;
416 : int fCountWriteOversize = 0;
417 : size_t fWriteMaxSize = 0;
418 : size_t fWriteMinSize = 0;
419 :
420 : // schema disabled by write error
421 : bool fDisabled = true;
422 :
423 : public:
424 :
425 0 : HsSchema() // ctor
426 0 : {
427 : // empty
428 0 : }
429 :
430 : virtual void remove_inactive_columns() = 0; // used by SQL schemas
431 : virtual void print(bool print_tags = true) const;
432 : virtual ~HsSchema(); // dtor
433 : virtual int flush_buffers() = 0;
434 : virtual int close() = 0;
435 : virtual int write_event(const time_t t, const char* data, const size_t data_size) = 0;
436 : virtual int match_event_var(const char* event_name, const char* var_name, const int var_index);
437 : virtual int read_last_written(const time_t timestamp,
438 : const int debug,
439 : time_t* last_written) = 0;
440 : virtual int read_data(const time_t start_time,
441 : const time_t end_time,
442 : const int num_var, const std::vector<int>& var_schema_index, const int var_index[],
443 : const int debug,
444 : std::vector<time_t>& last_time,
445 : MidasHistoryBufferInterface* buffer[]) = 0;
446 : };
447 :
448 : class HsSchemaVector
449 : {
450 : protected:
451 : std::vector<HsSchema*> fData;
452 :
453 : public:
454 0 : ~HsSchemaVector() { // dtor
455 0 : clear();
456 0 : }
457 :
458 0 : HsSchema* operator[](int index) const {
459 0 : return fData[index];
460 : }
461 :
462 0 : size_t size() const {
463 0 : return fData.size();
464 : }
465 :
466 : void add(HsSchema* s);
467 :
468 0 : void clear() {
469 0 : for (size_t i=0; i<fData.size(); i++)
470 0 : if (fData[i]) {
471 0 : delete fData[i];
472 0 : fData[i] = NULL;
473 : }
474 0 : fData.clear();
475 0 : }
476 :
477 0 : void print(bool print_tags = true) const {
478 0 : for (size_t i=0; i<fData.size(); i++)
479 0 : fData[i]->print(print_tags);
480 0 : }
481 :
482 : HsSchema* find_event(const char* event_name, const time_t timestamp, int debug = 0);
483 : };
484 :
485 : ////////////////////////////////////////////
486 : // Base class functions //
487 : ////////////////////////////////////////////
488 :
489 0 : HsSchema::~HsSchema() // dtor
490 : {
491 : //printf("HsSchema::dtor %p!\n", this);
492 : // only report if undersize/oversize happens more than once -
493 : // the first occurence is already reported by hs_write_event()
494 0 : if (fCountWriteUndersize > 1) {
495 0 : cm_msg(MERROR, "hs_write_event", "Event \'%s\' data size mismatch count: %d, expected %zu bytes, hs_write_event() called with as few as %zu bytes", fEventName.c_str(), fCountWriteUndersize, fNumBytes, fWriteMinSize);
496 : }
497 :
498 0 : if (fCountWriteOversize > 1) {
499 0 : cm_msg(MERROR, "hs_write_event", "Event \'%s\' data size mismatch count: %d, expected %zu bytes, hs_write_event() called with as many as %zu bytes", fEventName.c_str(), fCountWriteOversize, fNumBytes, fWriteMaxSize);
500 : }
501 0 : };
502 :
503 0 : void HsSchemaVector::add(HsSchema* s)
504 : {
505 : // schema list "data" is sorted by decreasing "fTimeFrom", newest schema first
506 :
507 : //printf("add: %s..%s %s\n", TimeToString(s->fTimeFrom).c_str(), TimeToString(s->fTimeTo).c_str(), s->fEventName.c_str());
508 :
509 0 : bool added = false;
510 :
511 0 : for (auto it = fData.begin(); it != fData.end(); it++) {
512 0 : if (event_name_cmp((*it)->fEventName, s->fEventName.c_str())==0) {
513 0 : if (s->fTimeFrom == (*it)->fTimeFrom) {
514 : // duplicate schema, keep the last one added (for file schema it is the newer file)
515 0 : s->fTimeTo = (*it)->fTimeTo;
516 0 : delete (*it);
517 0 : (*it) = s;
518 0 : return;
519 : }
520 : }
521 :
522 0 : if (s->fTimeFrom > (*it)->fTimeFrom) {
523 0 : fData.insert(it, s);
524 0 : added = true;
525 0 : break;
526 : }
527 : }
528 :
529 0 : if (!added) {
530 0 : fData.push_back(s);
531 : }
532 :
533 : //time_t oldest_time_from = fData.back()->fTimeFrom;
534 :
535 0 : time_t time_to = 0;
536 :
537 0 : for (auto it = fData.begin(); it != fData.end(); it++) {
538 0 : if (event_name_cmp((*it)->fEventName, s->fEventName.c_str())==0) {
539 0 : (*it)->fTimeTo = time_to;
540 0 : time_to = (*it)->fTimeFrom;
541 :
542 : //printf("vvv: %s..%s %s\n", TimeToString((*it)->fTimeFrom-oldest_time_from).c_str(), TimeToString((*it)->fTimeTo-oldest_time_from).c_str(), (*it)->fEventName.c_str());
543 : }
544 : }
545 : }
546 :
547 0 : HsSchema* HsSchemaVector::find_event(const char* event_name, time_t t, int debug)
548 : {
549 0 : HsSchema* ss = NULL;
550 :
551 0 : if (debug) {
552 0 : printf("find_event: All schema for event %s: (total %zu)\n", event_name, fData.size());
553 0 : int found = 0;
554 0 : for (size_t i=0; i<fData.size(); i++) {
555 0 : HsSchema* s = fData[i];
556 0 : printf("find_event: schema %zu name [%s]\n", i, s->fEventName.c_str());
557 0 : if (event_name)
558 0 : if (event_name_cmp(s->fEventName, event_name)!=0)
559 0 : continue;
560 0 : s->print();
561 0 : found++;
562 : }
563 0 : printf("find_event: Found %d schemas for event %s\n", found, event_name);
564 :
565 : //if (found == 0)
566 : // abort();
567 : }
568 :
569 0 : for (size_t i=0; i<fData.size(); i++) {
570 0 : HsSchema* s = fData[i];
571 :
572 : // wrong event
573 0 : if (event_name)
574 0 : if (event_name_cmp(s->fEventName, event_name)!=0)
575 0 : continue;
576 :
577 : // schema is from after the time we are looking for
578 0 : if (s->fTimeFrom > t)
579 0 : continue;
580 :
581 0 : if (!ss)
582 0 : ss = s;
583 :
584 : // remember the newest schema
585 0 : if (s->fTimeFrom > ss->fTimeFrom)
586 0 : ss = s;
587 : }
588 :
589 : // try to find
590 0 : for (size_t i=0; i<fData.size(); i++) {
591 0 : HsSchema* s = fData[i];
592 :
593 : // wrong event
594 0 : if (event_name)
595 0 : if (event_name_cmp(s->fEventName, event_name)!=0)
596 0 : continue;
597 :
598 : // schema is from after the time we are looking for
599 0 : if (s->fTimeFrom > t)
600 0 : continue;
601 :
602 0 : if (!ss)
603 0 : ss = s;
604 :
605 : // remember the newest schema
606 0 : if (s->fTimeFrom > ss->fTimeFrom)
607 0 : ss = s;
608 : }
609 :
610 0 : if (debug) {
611 0 : if (ss) {
612 0 : printf("find_event: for time %s, returning:\n", TimeToString(t).c_str());
613 0 : ss->print();
614 : } else {
615 0 : printf("find_event: for time %s, nothing found:\n", TimeToString(t).c_str());
616 : }
617 : }
618 :
619 0 : return ss;
620 : }
621 :
622 : ////////////////////////////////////////////
623 : // Sql interface class //
624 : ////////////////////////////////////////////
625 :
626 : class SqlBase
627 : {
628 : public:
629 : int fDebug = 0;
630 : bool fIsConnected = false;
631 : bool fTransactionPerTable = true;
632 :
633 0 : SqlBase() { // ctor
634 0 : };
635 :
636 0 : virtual ~SqlBase() { // dtor
637 : // confirm that the destructor of the concrete class
638 : // disconnected the database
639 0 : assert(!fIsConnected);
640 0 : fDebug = 0;
641 0 : fIsConnected = false;
642 0 : }
643 :
644 : virtual int Connect(const char* path) = 0;
645 : virtual int Disconnect() = 0;
646 : virtual bool IsConnected() = 0;
647 :
648 : virtual int ListTables(std::vector<std::string> *plist) = 0;
649 : virtual int ListColumns(const char* table_name, std::vector<std::string> *plist) = 0;
650 :
651 : // sql commands
652 : virtual int Exec(const char* table_name, const char* sql) = 0;
653 : virtual int ExecDisconnected(const char* table_name, const char* sql) = 0;
654 :
655 : // queries
656 : virtual int Prepare(const char* table_name, const char* sql) = 0;
657 : virtual int Step() = 0;
658 : virtual const char* GetText(int column) = 0;
659 : virtual time_t GetTime(int column) = 0;
660 : virtual double GetDouble(int column) = 0;
661 : virtual int Finalize() = 0;
662 :
663 : // transactions
664 : virtual int OpenTransaction(const char* table_name) = 0;
665 : virtual int CommitTransaction(const char* table_name) = 0;
666 : virtual int RollbackTransaction(const char* table_name) = 0;
667 :
668 : // data types
669 : virtual const char* ColumnType(int midas_tid) = 0;
670 : virtual bool TypesCompatible(int midas_tid, const char* sql_type) = 0;
671 :
672 : // string quoting
673 : virtual std::string QuoteString(const char* s) = 0; // quote text string
674 : virtual std::string QuoteId(const char* s) = 0; // quote identifier, such as table or column name
675 : };
676 :
677 : ////////////////////////////////////////////
678 : // Schema concrete classes //
679 : ////////////////////////////////////////////
680 :
681 : class HsSqlSchema : public HsSchema
682 : {
683 : public:
684 :
685 : SqlBase* fSql = NULL;
686 : std::string fTableName;
687 : std::vector<std::string> fColumnNames;
688 : std::vector<std::string> fColumnTypes;
689 : std::vector<bool> fColumnInactive;
690 :
691 : public:
692 :
693 0 : HsSqlSchema() // ctor
694 0 : {
695 : // empty
696 0 : }
697 :
698 0 : ~HsSqlSchema() // dtor
699 0 : {
700 0 : assert(get_transaction_count() == 0);
701 0 : }
702 :
703 : void remove_inactive_columns();
704 : void print(bool print_tags = true) const;
705 : int get_transaction_count();
706 : void reset_transaction_count();
707 : void increment_transaction_count();
708 : int close_transaction();
709 0 : int flush_buffers() { return close_transaction(); }
710 0 : int close() { return close_transaction(); }
711 : int write_event(const time_t t, const char* data, const size_t data_size);
712 : int match_event_var(const char* event_name, const char* var_name, const int var_index);
713 : int read_last_written(const time_t timestamp,
714 : const int debug,
715 : time_t* last_written);
716 : int read_data(const time_t start_time,
717 : const time_t end_time,
718 : const int num_var, const std::vector<int>& var_schema_index, const int var_index[],
719 : const int debug,
720 : std::vector<time_t>& last_time,
721 : MidasHistoryBufferInterface* buffer[]);
722 :
723 : private:
724 : // Sqlite uses a transaction per table; MySQL uses a single transaction for all tables.
725 : // But to support future "single transaction" DBs more easily (e.g. if user wants to
726 : // log to both Postgres and MySQL in future), we keep track of the transaction count
727 : // per SQL engine.
728 : int fTableTransactionCount = 0;
729 : static std::map<SqlBase*, int> gfTransactionCount;
730 : };
731 :
732 : std::map<SqlBase*, int> HsSqlSchema::gfTransactionCount;
733 :
734 : class HsFileSchema : public HsSchema
735 : {
736 : public:
737 :
738 : std::string fFileName;
739 : size_t fRecordSize = 0;
740 : off64_t fDataOffset = 0;
741 : size_t fLastSize = 0;
742 : int fWriterFd = -1;
743 : size_t fRecordBufferSize = 0;
744 : char* fRecordBuffer = NULL;
745 :
746 : public:
747 : off64_t fFileSizeInitial = 0; // initial file size
748 : off64_t fFileSize = 0; // file size including any new data we wrote
749 :
750 : public:
751 :
752 0 : HsFileSchema() // ctor
753 0 : {
754 : // empty
755 0 : }
756 :
757 0 : ~HsFileSchema() // dtor
758 0 : {
759 : //printf("HsFileSchema::dtor %p!\n", this);
760 0 : close();
761 0 : fRecordSize = 0;
762 0 : fDataOffset = 0;
763 0 : fLastSize = 0;
764 0 : fWriterFd = -1;
765 0 : if (fRecordBuffer) {
766 0 : free(fRecordBuffer);
767 0 : fRecordBuffer = NULL;
768 : }
769 0 : fRecordBufferSize = 0;
770 0 : }
771 :
772 0 : void remove_inactive_columns() { /* empty */ };
773 : void print(bool print_tags = true) const;
774 0 : int flush_buffers() { return HS_SUCCESS; };
775 : int close();
776 : int write_event(const time_t t, const char* data, const size_t data_size);
777 : int read_last_written(const time_t timestamp,
778 : const int debug,
779 : time_t* last_written);
780 : int read_data(const time_t start_time,
781 : const time_t end_time,
782 : const int num_var, const std::vector<int>& var_schema_index, const int var_index[],
783 : const int debug,
784 : std::vector<time_t>& last_time,
785 : MidasHistoryBufferInterface* buffer[]);
786 : };
787 :
788 : ////////////////////////////////////////////
789 : // Print functions //
790 : ////////////////////////////////////////////
791 :
792 0 : void HsSchema::print(bool print_tags) const
793 : {
794 0 : size_t nv = this->fVariables.size();
795 0 : printf("event [%s], time %s..%s, %zu variables, %zu bytes\n", this->fEventName.c_str(), TimeToString(this->fTimeFrom).c_str(), TimeToString(this->fTimeTo).c_str(), nv, fNumBytes);
796 0 : if (print_tags)
797 0 : for (size_t j=0; j<nv; j++)
798 0 : printf(" %zu: name [%s], type [%s] tid %d, n_data %d, n_bytes %d, offset %d\n", j, this->fVariables[j].name.c_str(), rpc_tid_name(this->fVariables[j].type), this->fVariables[j].type, this->fVariables[j].n_data, this->fVariables[j].n_bytes, this->fOffsets[j]);
799 0 : };
800 :
801 0 : void HsSqlSchema::print(bool print_tags) const
802 : {
803 0 : size_t nv = this->fVariables.size();
804 0 : printf("event [%s], sql_table [%s], time %s..%s, %zu variables, %zu bytes\n", this->fEventName.c_str(), this->fTableName.c_str(), TimeToString(this->fTimeFrom).c_str(), TimeToString(this->fTimeTo).c_str(), nv, fNumBytes);
805 0 : if (print_tags) {
806 0 : for (size_t j=0; j<nv; j++) {
807 0 : printf(" %zu: name [%s], type [%s] tid %d, n_data %d, n_bytes %d", j, this->fVariables[j].name.c_str(), rpc_tid_name(this->fVariables[j].type), this->fVariables[j].type, this->fVariables[j].n_data, this->fVariables[j].n_bytes);
808 0 : printf(", sql_column [%s], sql_type [%s], offset %d", this->fColumnNames[j].c_str(), this->fColumnTypes[j].c_str(), this->fOffsets[j]);
809 0 : printf(", inactive %d", (int)this->fColumnInactive[j]);
810 0 : printf("\n");
811 : }
812 : }
813 0 : }
814 :
815 0 : void HsFileSchema::print(bool print_tags) const
816 : {
817 0 : size_t nv = this->fVariables.size();
818 0 : printf("event [%s], file_name [%s], time %s..%s, %zu variables, %zu bytes, dat_offset %jd, record_size %zu\n", this->fEventName.c_str(), this->fFileName.c_str(), TimeToString(this->fTimeFrom).c_str(), TimeToString(this->fTimeTo).c_str(), nv, fNumBytes, (intmax_t)fDataOffset, fRecordSize);
819 0 : if (print_tags) {
820 0 : for (size_t j=0; j<nv; j++)
821 0 : printf(" %zu: name [%s], type [%s] tid %d, n_data %d, n_bytes %d, offset %d\n", j, this->fVariables[j].name.c_str(), rpc_tid_name(this->fVariables[j].type), this->fVariables[j].type, this->fVariables[j].n_data, this->fVariables[j].n_bytes, this->fOffsets[j]);
822 : }
823 0 : }
824 :
825 : ////////////////////////////////////////////
826 : // File functions //
827 : ////////////////////////////////////////////
828 :
829 : #ifdef HAVE_MYSQL
830 :
831 : ////////////////////////////////////////
832 : // MYSQL/MariaDB database access //
833 : ////////////////////////////////////////
834 :
835 : //#warning !!!HAVE_MYSQL!!!
836 :
837 : //#include <my_global.h> // my_global.h removed MySQL 8.0, MariaDB 10.2. K.O.
838 : #include <mysql.h>
839 :
840 : class Mysql: public SqlBase
841 : {
842 : public:
843 : std::string fConnectString;
844 : MYSQL* fMysql = NULL;
845 :
846 : // query results
847 : MYSQL_RES* fResult = NULL;
848 : MYSQL_ROW fRow = NULL;
849 : int fNumFields = 0;
850 :
851 : // disconnected operation
852 : size_t fMaxDisconnected = 0;
853 : std::list<std::string> fDisconnectedBuffer;
854 : time_t fNextReconnect = 0;
855 : int fNextReconnectDelaySec = 0;
856 : int fDisconnectedLost = 0;
857 :
858 : Mysql(); // ctor
859 : ~Mysql(); // dtor
860 :
861 : int Connect(const char* path);
862 : int Disconnect();
863 : bool IsConnected();
864 :
865 : int ConnectTable(const char* table_name);
866 :
867 : int ListTables(std::vector<std::string> *plist);
868 : int ListColumns(const char* table_name, std::vector<std::string> *plist);
869 :
870 : int Exec(const char* table_name, const char* sql);
871 : int ExecDisconnected(const char* table_name, const char* sql);
872 :
873 : int Prepare(const char* table_name, const char* sql);
874 : int Step();
875 : const char* GetText(int column);
876 : time_t GetTime(int column);
877 : double GetDouble(int column);
878 : int Finalize();
879 :
880 : int OpenTransaction(const char* table_name);
881 : int CommitTransaction(const char* table_name);
882 : int RollbackTransaction(const char* table_name);
883 :
884 : const char* ColumnType(int midas_tid);
885 : bool TypesCompatible(int midas_tid, const char* sql_type);
886 :
887 : std::string QuoteId(const char* s);
888 : std::string QuoteString(const char* s);
889 : };
890 :
891 0 : Mysql::Mysql() // ctor
892 : {
893 0 : fMysql = NULL;
894 0 : fResult = NULL;
895 0 : fRow = NULL;
896 0 : fNumFields = 0;
897 0 : fMaxDisconnected = 1000;
898 0 : fNextReconnect = 0;
899 0 : fNextReconnectDelaySec = 0;
900 0 : fDisconnectedLost = 0;
901 0 : fTransactionPerTable = false;
902 0 : }
903 :
904 0 : Mysql::~Mysql() // dtor
905 : {
906 0 : Disconnect();
907 0 : fMysql = NULL;
908 0 : fResult = NULL;
909 0 : fRow = NULL;
910 0 : fNumFields = 0;
911 0 : if (fDisconnectedBuffer.size() > 0) {
912 0 : cm_msg(MINFO, "Mysql::~Mysql", "Lost %zu history entries accumulated while disconnected from the database", fDisconnectedBuffer.size());
913 0 : cm_msg_flush_buffer();
914 : }
915 0 : }
916 :
917 0 : int Mysql::Connect(const char* connect_string)
918 : {
919 0 : if (fIsConnected)
920 0 : Disconnect();
921 :
922 0 : fConnectString = connect_string;
923 :
924 0 : if (fDebug) {
925 0 : cm_msg(MINFO, "Mysql::Connect", "Connecting to Mysql database specified by \'%s\'", connect_string);
926 0 : cm_msg_flush_buffer();
927 : }
928 :
929 0 : std::string host_name;
930 0 : std::string user_name;
931 0 : std::string user_password;
932 0 : std::string db_name;
933 0 : int tcp_port = 0;
934 0 : std::string unix_socket;
935 0 : std::string buffer;
936 :
937 0 : FILE* fp = fopen(connect_string, "r");
938 0 : if (!fp) {
939 0 : cm_msg(MERROR, "Mysql::Connect", "Cannot read MYSQL connection parameters from \'%s\', fopen() error %d (%s)", connect_string, errno, strerror(errno));
940 0 : return DB_FILE_ERROR;
941 : }
942 :
943 : while (1) {
944 : char buf[256];
945 0 : char* s = fgets(buf, sizeof(buf), fp);
946 0 : if (!s)
947 0 : break; // EOF
948 :
949 : char*ss;
950 : // kill trailing \n and \r
951 0 : ss = strchr(s, '\n');
952 0 : if (ss) *ss = 0;
953 0 : ss = strchr(s, '\r');
954 0 : if (ss) *ss = 0;
955 :
956 : //printf("line [%s]\n", s);
957 :
958 0 : if (strncasecmp(s, "server=", 7)==0)
959 0 : host_name = skip_spaces(s + 7);
960 0 : if (strncasecmp(s, "port=", 5)==0)
961 0 : tcp_port = atoi(skip_spaces(s + 5));
962 0 : if (strncasecmp(s, "database=", 9)==0)
963 0 : db_name = skip_spaces(s + 9);
964 0 : if (strncasecmp(s, "socket=", 7)==0)
965 0 : unix_socket = skip_spaces(s + 7);
966 0 : if (strncasecmp(s, "user=", 5)==0)
967 0 : user_name = skip_spaces(s + 5);
968 0 : if (strncasecmp(s, "password=", 9)==0)
969 0 : user_password = skip_spaces(s + 9);
970 0 : if (strncasecmp(s, "buffer=", 7)==0)
971 0 : buffer = skip_spaces(s + 7);
972 0 : }
973 :
974 0 : fclose(fp);
975 :
976 0 : int buffer_int = atoi(buffer.c_str());
977 :
978 0 : if (buffer_int > 0 && buffer_int < 1000000)
979 0 : fMaxDisconnected = buffer_int;
980 :
981 0 : if (fDebug)
982 0 : printf("Mysql::Connect: connecting to server [%s] port %d, unix socket [%s], database [%s], user [%s], password [%s], buffer [%zu]\n", host_name.c_str(), tcp_port, unix_socket.c_str(), db_name.c_str(), user_name.c_str(), user_password.c_str(), fMaxDisconnected);
983 :
984 0 : if (!fMysql) {
985 0 : fMysql = mysql_init(NULL);
986 0 : if (!fMysql) {
987 0 : return DB_FILE_ERROR;
988 : }
989 : }
990 :
991 0 : int client_flag = 0|CLIENT_IGNORE_SIGPIPE;
992 :
993 0 : if (mysql_real_connect(fMysql, host_name.c_str(), user_name.c_str(), user_password.c_str(), db_name.c_str(), tcp_port, unix_socket.c_str(), client_flag) == NULL) {
994 0 : cm_msg(MERROR, "Mysql::Connect", "mysql_real_connect() to host [%s], port %d, unix socket [%s], database [%s], user [%s], password [%s]: error %d (%s)", host_name.c_str(), tcp_port, unix_socket.c_str(), db_name.c_str(), user_name.c_str(), "xxx", mysql_errno(fMysql), mysql_error(fMysql));
995 0 : Disconnect();
996 0 : return DB_FILE_ERROR;
997 : }
998 :
999 : int status;
1000 :
1001 : // FIXME:
1002 : //my_bool reconnect = 0;
1003 : //mysql_options(&mysql, MYSQL_OPT_RECONNECT, &reconnect);
1004 :
1005 0 : status = Exec("(notable)", "SET SESSION sql_mode='ANSI'");
1006 0 : if (status != DB_SUCCESS) {
1007 0 : cm_msg(MERROR, "Mysql::Connect", "Cannot set ANSI mode, nothing will work");
1008 0 : Disconnect();
1009 0 : return DB_FILE_ERROR;
1010 : }
1011 :
1012 0 : if (fDebug) {
1013 0 : cm_msg(MINFO, "Mysql::Connect", "Connected to a MySQL database on host [%s], port %d, unix socket [%s], database [%s], user [%s], password [%s], buffer %zu", host_name.c_str(), tcp_port, unix_socket.c_str(), db_name.c_str(), user_name.c_str(), "xxx", fMaxDisconnected);
1014 0 : cm_msg_flush_buffer();
1015 : }
1016 :
1017 0 : fIsConnected = true;
1018 :
1019 0 : int count = 0;
1020 0 : while (fDisconnectedBuffer.size() > 0) {
1021 0 : status = Exec("(flush)", fDisconnectedBuffer.front().c_str());
1022 0 : if (status != DB_SUCCESS) {
1023 0 : return status;
1024 : }
1025 0 : fDisconnectedBuffer.pop_front();
1026 0 : count++;
1027 : }
1028 :
1029 0 : if (count > 0) {
1030 0 : cm_msg(MINFO, "Mysql::Connect", "Saved %d, lost %d history events accumulated while disconnected from the database", count, fDisconnectedLost);
1031 0 : cm_msg_flush_buffer();
1032 : }
1033 :
1034 0 : assert(fDisconnectedBuffer.size() == 0);
1035 0 : fDisconnectedLost = 0;
1036 :
1037 0 : return DB_SUCCESS;
1038 0 : }
1039 :
1040 0 : int Mysql::Disconnect()
1041 : {
1042 0 : if (fRow) {
1043 : // FIXME: mysql_free_result(fResult);
1044 : }
1045 :
1046 0 : if (fResult)
1047 0 : mysql_free_result(fResult);
1048 :
1049 0 : if (fMysql)
1050 0 : mysql_close(fMysql);
1051 :
1052 0 : fMysql = NULL;
1053 0 : fResult = NULL;
1054 0 : fRow = NULL;
1055 :
1056 0 : fIsConnected = false;
1057 0 : return DB_SUCCESS;
1058 : }
1059 :
1060 0 : bool Mysql::IsConnected()
1061 : {
1062 0 : return fIsConnected;
1063 : }
1064 :
1065 0 : int Mysql::OpenTransaction(const char* table_name)
1066 : {
1067 0 : return Exec(table_name, "START TRANSACTION");
1068 : return DB_SUCCESS;
1069 : }
1070 :
1071 0 : int Mysql::CommitTransaction(const char* table_name)
1072 : {
1073 0 : Exec(table_name, "COMMIT");
1074 0 : return DB_SUCCESS;
1075 : }
1076 :
1077 0 : int Mysql::RollbackTransaction(const char* table_name)
1078 : {
1079 0 : Exec(table_name, "ROLLBACK");
1080 0 : return DB_SUCCESS;
1081 : }
1082 :
1083 0 : int Mysql::ListTables(std::vector<std::string> *plist)
1084 : {
1085 0 : if (!fIsConnected)
1086 0 : return DB_FILE_ERROR;
1087 :
1088 0 : if (fDebug)
1089 0 : printf("Mysql::ListTables!\n");
1090 :
1091 : int status;
1092 :
1093 0 : fResult = mysql_list_tables(fMysql, NULL);
1094 :
1095 0 : if (fResult == NULL) {
1096 0 : cm_msg(MERROR, "Mysql::ListTables", "mysql_list_tables() error %d (%s)", mysql_errno(fMysql), mysql_error(fMysql));
1097 0 : return DB_FILE_ERROR;
1098 : }
1099 :
1100 0 : fNumFields = mysql_num_fields(fResult);
1101 :
1102 : while (1) {
1103 0 : status = Step();
1104 0 : if (status != DB_SUCCESS)
1105 0 : break;
1106 0 : std::string tn = GetText(0);
1107 0 : plist->push_back(tn);
1108 0 : };
1109 :
1110 0 : status = Finalize();
1111 :
1112 0 : return DB_SUCCESS;
1113 : }
1114 :
1115 0 : int Mysql::ListColumns(const char* table_name, std::vector<std::string> *plist)
1116 : {
1117 0 : if (!fIsConnected)
1118 0 : return DB_FILE_ERROR;
1119 :
1120 0 : if (fDebug)
1121 0 : printf("Mysql::ListColumns for table \'%s\'\n", table_name);
1122 :
1123 : int status;
1124 :
1125 0 : std::string cmd;
1126 0 : cmd += "SHOW COLUMNS FROM ";
1127 0 : cmd += QuoteId(table_name);
1128 0 : cmd += ";";
1129 :
1130 0 : status = Prepare(table_name, cmd.c_str());
1131 0 : if (status != DB_SUCCESS)
1132 0 : return status;
1133 :
1134 0 : fNumFields = mysql_num_fields(fResult);
1135 :
1136 : while (1) {
1137 0 : status = Step();
1138 0 : if (status != DB_SUCCESS)
1139 0 : break;
1140 0 : std::string cn = GetText(0);
1141 0 : std::string ct = GetText(1);
1142 0 : plist->push_back(cn);
1143 0 : plist->push_back(ct);
1144 : //printf("cn [%s]\n", cn.c_str());
1145 : //for (int i=0; i<fNumFields; i++)
1146 : //printf(" field[%d]: [%s]\n", i, GetText(i));
1147 0 : };
1148 :
1149 0 : status = Finalize();
1150 :
1151 0 : return DB_SUCCESS;
1152 0 : }
1153 :
1154 0 : int Mysql::Exec(const char* table_name, const char* sql)
1155 : {
1156 0 : if (fDebug)
1157 0 : printf("Mysql::Exec(%s, %s)\n", table_name, sql);
1158 :
1159 : // FIXME: match Sqlite::Exec() return values:
1160 : // return values:
1161 : // DB_SUCCESS
1162 : // DB_FILE_ERROR: not connected
1163 : // DB_KEY_EXIST: "table already exists"
1164 :
1165 0 : if (!fMysql)
1166 0 : return DB_FILE_ERROR;
1167 :
1168 0 : assert(fMysql);
1169 0 : assert(fResult == NULL); // there should be no unfinalized queries
1170 0 : assert(fRow == NULL);
1171 :
1172 0 : if (mysql_query(fMysql, sql)) {
1173 0 : if (mysql_errno(fMysql) == 1050) { // "Table already exists"
1174 0 : return DB_KEY_EXIST;
1175 : }
1176 0 : if (mysql_errno(fMysql) == 1146) { // "Table does not exist"
1177 0 : return DB_FILE_ERROR;
1178 : }
1179 0 : cm_msg(MERROR, "Mysql::Exec", "mysql_query(%s) error %d (%s)", sql, mysql_errno(fMysql), mysql_error(fMysql));
1180 0 : if (mysql_errno(fMysql) == 1060) // "Duplicate column name"
1181 0 : return DB_KEY_EXIST;
1182 0 : if (mysql_errno(fMysql) == 2006) { // "MySQL server has gone away"
1183 0 : Disconnect();
1184 0 : return ExecDisconnected(table_name, sql);
1185 : }
1186 0 : return DB_FILE_ERROR;
1187 : }
1188 :
1189 0 : return DB_SUCCESS;
1190 : }
1191 :
1192 0 : int Mysql::ExecDisconnected(const char* table_name, const char* sql)
1193 : {
1194 0 : if (fDebug)
1195 0 : printf("Mysql::ExecDisconnected(%s, %s)\n", table_name, sql);
1196 :
1197 0 : if (fDisconnectedBuffer.size() < fMaxDisconnected) {
1198 0 : fDisconnectedBuffer.push_back(sql);
1199 0 : if (fDisconnectedBuffer.size() >= fMaxDisconnected) {
1200 0 : cm_msg(MERROR, "Mysql::ExecDisconnected", "Error: Disconnected database buffer overflow, size %zu, subsequent events are lost", fDisconnectedBuffer.size());
1201 : }
1202 : } else {
1203 0 : fDisconnectedLost++;
1204 : }
1205 :
1206 0 : time_t now = time(NULL);
1207 :
1208 0 : if (fNextReconnect == 0 || now >= fNextReconnect) {
1209 0 : int status = Connect(fConnectString.c_str());
1210 0 : if (status == DB_SUCCESS) {
1211 0 : fNextReconnect = 0;
1212 0 : fNextReconnectDelaySec = 0;
1213 : } else {
1214 0 : if (fNextReconnectDelaySec == 0) {
1215 0 : fNextReconnectDelaySec = 5;
1216 0 : } else if (fNextReconnectDelaySec < 10*60) {
1217 0 : fNextReconnectDelaySec *= 2;
1218 : }
1219 0 : if (fDebug) {
1220 0 : cm_msg(MINFO, "Mysql::ExecDisconnected", "Next reconnect attempt in %d sec, history events buffered %zu, lost %d", fNextReconnectDelaySec, fDisconnectedBuffer.size(), fDisconnectedLost);
1221 0 : cm_msg_flush_buffer();
1222 : }
1223 0 : fNextReconnect = now + fNextReconnectDelaySec;
1224 : }
1225 : }
1226 :
1227 0 : return DB_SUCCESS;
1228 : }
1229 :
1230 0 : int Mysql::Prepare(const char* table_name, const char* sql)
1231 : {
1232 0 : if (fDebug)
1233 0 : printf("Mysql::Prepare(%s, %s)\n", table_name, sql);
1234 :
1235 0 : if (!fMysql)
1236 0 : return DB_FILE_ERROR;
1237 :
1238 0 : assert(fMysql);
1239 0 : assert(fResult == NULL); // there should be no unfinalized queries
1240 0 : assert(fRow == NULL);
1241 :
1242 : // if (mysql_query(fMysql, sql)) {
1243 : // cm_msg(MERROR, "Mysql::Prepare", "mysql_query(%s) error %d (%s)", sql, mysql_errno(fMysql), mysql_error(fMysql));
1244 : // return DB_FILE_ERROR;
1245 : //}
1246 :
1247 : // Check if the connection to MySQL timed out; fix from B. Smith
1248 0 : int status = mysql_query(fMysql, sql);
1249 0 : if (status) {
1250 0 : if (mysql_errno(fMysql) == 2006 || mysql_errno(fMysql) == 2013) {
1251 : // "MySQL server has gone away" or "Lost connection to MySQL server during query"
1252 0 : status = Connect(fConnectString.c_str());
1253 0 : if (status == DB_SUCCESS) {
1254 : // Retry after reconnecting
1255 0 : status = mysql_query(fMysql, sql);
1256 : } else {
1257 0 : cm_msg(MERROR, "Mysql::Prepare", "mysql_query(%s) - MySQL server has gone away, and couldn't reconnect - %d", sql, status);
1258 0 : return DB_FILE_ERROR;
1259 : }
1260 : }
1261 0 : if (status) {
1262 0 : cm_msg(MERROR, "Mysql::Prepare", "mysql_query(%s) error %d (%s)", sql, mysql_errno(fMysql), mysql_error(fMysql));
1263 0 : return DB_FILE_ERROR;
1264 : }
1265 0 : cm_msg(MINFO, "Mysql::Prepare", "Reconnected to MySQL after long inactivity.");
1266 : }
1267 :
1268 0 : fResult = mysql_store_result(fMysql);
1269 : //fResult = mysql_use_result(fMysql); // cannot use this because it blocks writing into table
1270 :
1271 0 : if (!fResult) {
1272 0 : cm_msg(MERROR, "Mysql::Prepare", "mysql_store_result(%s) returned NULL, error %d (%s)", sql, mysql_errno(fMysql), mysql_error(fMysql));
1273 0 : return DB_FILE_ERROR;
1274 : }
1275 :
1276 0 : fNumFields = mysql_num_fields(fResult);
1277 :
1278 : //printf("num fields %d\n", fNumFields);
1279 :
1280 0 : return DB_SUCCESS;
1281 : }
1282 :
1283 0 : int Mysql::Step()
1284 : {
1285 : if (/* DISABLES CODE */ (0) && fDebug)
1286 : printf("Mysql::Step()\n");
1287 :
1288 0 : assert(fMysql);
1289 0 : assert(fResult);
1290 :
1291 0 : fRow = mysql_fetch_row(fResult);
1292 :
1293 0 : if (fRow)
1294 0 : return DB_SUCCESS;
1295 :
1296 0 : if (mysql_errno(fMysql) == 0)
1297 0 : return DB_NO_MORE_SUBKEYS;
1298 :
1299 0 : cm_msg(MERROR, "Mysql::Step", "mysql_fetch_row() error %d (%s)", mysql_errno(fMysql), mysql_error(fMysql));
1300 :
1301 0 : return DB_FILE_ERROR;
1302 : }
1303 :
1304 0 : const char* Mysql::GetText(int column)
1305 : {
1306 0 : assert(fMysql);
1307 0 : assert(fResult);
1308 0 : assert(fRow);
1309 0 : assert(fNumFields > 0);
1310 0 : assert(column >= 0);
1311 0 : assert(column < fNumFields);
1312 0 : if (fRow[column] == NULL)
1313 0 : return "";
1314 0 : return fRow[column];
1315 : }
1316 :
1317 0 : double Mysql::GetDouble(int column)
1318 : {
1319 0 : return atof(GetText(column));
1320 : }
1321 :
1322 0 : time_t Mysql::GetTime(int column)
1323 : {
1324 0 : return strtoul(GetText(column), NULL, 0);
1325 : }
1326 :
1327 0 : int Mysql::Finalize()
1328 : {
1329 0 : assert(fMysql);
1330 0 : assert(fResult);
1331 :
1332 0 : mysql_free_result(fResult);
1333 0 : fResult = NULL;
1334 0 : fRow = NULL;
1335 0 : fNumFields = 0;
1336 :
1337 0 : return DB_SUCCESS;
1338 : }
1339 :
1340 0 : const char* Mysql::ColumnType(int midas_tid)
1341 : {
1342 0 : assert(midas_tid>=0);
1343 0 : assert(midas_tid<TID_LAST);
1344 0 : return sql_type_mysql[midas_tid];
1345 : }
1346 :
1347 0 : bool Mysql::TypesCompatible(int midas_tid, const char* sql_type)
1348 : {
1349 : if (/* DISABLES CODE */ (0))
1350 : printf("compare types midas \'%s\'=\'%s\' and sql \'%s\'\n", rpc_tid_name(midas_tid), ColumnType(midas_tid), sql_type);
1351 :
1352 : //if (sql2midasType_mysql(sql_type) == midas_tid)
1353 : // return true;
1354 :
1355 0 : if (strcasecmp(ColumnType(midas_tid), sql_type) == 0)
1356 0 : return true;
1357 :
1358 : // permit writing FLOAT into DOUBLE
1359 0 : if (midas_tid==TID_FLOAT && strcmp(sql_type, "double")==0)
1360 0 : return true;
1361 :
1362 : // T2K quirk!
1363 : // permit writing BYTE into signed tinyint
1364 0 : if (midas_tid==TID_BYTE && strcmp(sql_type, "tinyint")==0)
1365 0 : return true;
1366 :
1367 : // T2K quirk!
1368 : // permit writing WORD into signed tinyint
1369 0 : if (midas_tid==TID_WORD && strcmp(sql_type, "tinyint")==0)
1370 0 : return true;
1371 :
1372 : // mysql quirk!
1373 : //if (midas_tid==TID_DWORD && strcmp(sql_type, "int(10) unsigned")==0)
1374 : // return true;
1375 :
1376 : if (/* DISABLES CODE */ (0))
1377 : printf("type mismatch!\n");
1378 :
1379 0 : return false;
1380 : }
1381 :
1382 0 : std::string Mysql::QuoteId(const char* s)
1383 : {
1384 0 : std::string q;
1385 0 : q += "`";
1386 0 : q += s;
1387 0 : q += "`";
1388 0 : return q;
1389 0 : }
1390 :
1391 0 : std::string Mysql::QuoteString(const char* s)
1392 : {
1393 0 : std::string q;
1394 0 : q += "\'";
1395 0 : q += s;
1396 : #if 0
1397 : while (int c = *s++) {
1398 : if (c == '\'') {
1399 : q += "\\'";
1400 : } if (c == '"') {
1401 : q += "\\\"";
1402 : } else if (isprint(c)) {
1403 : q += c;
1404 : } else {
1405 : char buf[256];
1406 : sprintf(buf, "\\\\x%02x", c&0xFF);
1407 : q += buf;
1408 : }
1409 : }
1410 : #endif
1411 0 : q += "\'";
1412 0 : return q;
1413 0 : }
1414 :
1415 : #endif // HAVE_MYSQL
1416 :
1417 : #ifdef HAVE_PGSQL
1418 :
1419 : ////////////////////////////////////////
1420 : // PostgreSQL database access //
1421 : ////////////////////////////////////////
1422 :
1423 : //#warning !!!HAVE_PGSQL!!!
1424 :
1425 : #include <libpq-fe.h>
1426 :
1427 : class Pgsql: public SqlBase
1428 : {
1429 : public:
1430 : std::string fConnectString;
1431 : int fDownsample = 0;
1432 : PGconn* fPgsql = NULL;
1433 :
1434 : // query results
1435 : PGresult *fResult = NULL;
1436 : int fNumFields = 0;
1437 : int fRow = 0;
1438 :
1439 : // disconnected operation
1440 : size_t fMaxDisconnected = 0;
1441 : std::list<std::string> fDisconnectedBuffer;
1442 : time_t fNextReconnect = 0;
1443 : int fNextReconnectDelaySec = 0;
1444 : int fDisconnectedLost = 0;
1445 :
1446 : Pgsql(); // ctor
1447 : ~Pgsql(); // dtor
1448 :
1449 : int Connect(const char* path);
1450 : int Disconnect();
1451 : bool IsConnected();
1452 :
1453 : int ConnectTable(const char* table_name);
1454 :
1455 : int ListTables(std::vector<std::string> *plist);
1456 : int ListColumns(const char* table_name, std::vector<std::string> *plist);
1457 :
1458 : int Exec(const char* table_name, const char* sql);
1459 : int ExecDisconnected(const char* table_name, const char* sql);
1460 :
1461 : int Prepare(const char* table_name, const char* sql);
1462 : std::string BuildDownsampleQuery(const time_t start_time, const time_t end_time, const int npoints, const char* table_name, const char* column_name);
1463 : int Step();
1464 : const char* GetText(int column);
1465 : time_t GetTime(int column);
1466 : double GetDouble(int column);
1467 : int Finalize();
1468 :
1469 : int OpenTransaction(const char* table_name);
1470 : int CommitTransaction(const char* table_name);
1471 : int RollbackTransaction(const char* table_name);
1472 :
1473 : const char* ColumnType(int midas_tid);
1474 : bool TypesCompatible(int midas_tid, const char* sql_type);
1475 :
1476 : std::string QuoteId(const char* s);
1477 : std::string QuoteString(const char* s);
1478 : };
1479 :
1480 0 : Pgsql::Pgsql() // ctor
1481 : {
1482 0 : fPgsql = NULL;
1483 0 : fDownsample = 0;
1484 0 : fResult = NULL;
1485 0 : fRow = -1;
1486 0 : fNumFields = 0;
1487 0 : fMaxDisconnected = 1000;
1488 0 : fNextReconnect = 0;
1489 0 : fNextReconnectDelaySec = 0;
1490 0 : fDisconnectedLost = 0;
1491 0 : fTransactionPerTable = false;
1492 0 : }
1493 :
1494 0 : Pgsql::~Pgsql() // dtor
1495 : {
1496 0 : Disconnect();
1497 0 : if(fResult)
1498 0 : PQclear(fResult);
1499 0 : fRow = -1;
1500 0 : fNumFields = 0;
1501 0 : if (fDisconnectedBuffer.size() > 0) {
1502 0 : cm_msg(MINFO, "Pgsql::~Pgsql", "Lost %zu history entries accumulated while disconnected from the database", fDisconnectedBuffer.size());
1503 0 : cm_msg_flush_buffer();
1504 : }
1505 0 : }
1506 :
1507 0 : int Pgsql::Connect(const char* connect_string)
1508 : {
1509 0 : if (fIsConnected)
1510 0 : Disconnect();
1511 :
1512 0 : fConnectString = connect_string;
1513 :
1514 0 : if (fDebug) {
1515 0 : cm_msg(MINFO, "Pgsql::Connect", "Connecting to PostgreSQL database specified by \'%s\'", connect_string);
1516 0 : cm_msg_flush_buffer();
1517 : }
1518 :
1519 0 : std::string host_name;
1520 0 : std::string user_name;
1521 0 : std::string user_password;
1522 0 : std::string db_name;
1523 0 : std::string tcp_port;
1524 0 : std::string unix_socket;
1525 0 : std::string buffer;
1526 :
1527 0 : FILE* fp = fopen(connect_string, "r");
1528 0 : if (!fp) {
1529 0 : cm_msg(MERROR, "Pgsql::Connect", "Cannot read PostgreSQL connection parameters from \'%s\', fopen() error %d (%s)", connect_string, errno, strerror(errno));
1530 0 : return DB_FILE_ERROR;
1531 : }
1532 :
1533 : while (1) {
1534 : char buf[256];
1535 0 : char* s = fgets(buf, sizeof(buf), fp);
1536 0 : if (!s)
1537 0 : break; // EOF
1538 :
1539 : char*ss;
1540 : // kill trailing \n and \r
1541 0 : ss = strchr(s, '\n');
1542 0 : if (ss) *ss = 0;
1543 0 : ss = strchr(s, '\r');
1544 0 : if (ss) *ss = 0;
1545 :
1546 : //printf("line [%s]\n", s);
1547 :
1548 0 : if (strncasecmp(s, "server=", 7)==0)
1549 0 : host_name = skip_spaces(s + 7);
1550 0 : if (strncasecmp(s, "port=", 5)==0)
1551 0 : tcp_port = skip_spaces(s + 5);
1552 0 : if (strncasecmp(s, "database=", 9)==0)
1553 0 : db_name = skip_spaces(s + 9);
1554 0 : if (strncasecmp(s, "socket=", 7)==0)
1555 0 : unix_socket = skip_spaces(s + 7);
1556 0 : if (strncasecmp(s, "user=", 5)==0)
1557 0 : user_name = skip_spaces(s + 5);
1558 0 : if (strncasecmp(s, "password=", 9)==0)
1559 0 : user_password = skip_spaces(s + 9);
1560 0 : if (strncasecmp(s, "buffer=", 7)==0)
1561 0 : buffer = skip_spaces(s + 7);
1562 0 : }
1563 :
1564 0 : fclose(fp);
1565 :
1566 0 : int buffer_int = atoi(buffer.c_str());
1567 :
1568 0 : if (buffer_int > 0 && buffer_int < 1000000)
1569 0 : fMaxDisconnected = buffer_int;
1570 :
1571 0 : if (fDebug)
1572 0 : printf("Pgsql::Connect: connecting to server [%s] port %s, unix socket [%s], database [%s], user [%s], password [%s], buffer [%zu]\n", host_name.c_str(), tcp_port.c_str(), unix_socket.c_str(), db_name.c_str(), user_name.c_str(), user_password.c_str(), fMaxDisconnected);
1573 :
1574 0 : fPgsql = PQsetdbLogin(host_name.c_str(), tcp_port.c_str(), NULL, NULL, db_name.c_str(), user_name.c_str(), user_password.c_str());
1575 0 : if (PQstatus(fPgsql) != CONNECTION_OK) {
1576 0 : std::string msg(PQerrorMessage(fPgsql));
1577 0 : msg.erase(std::remove(msg.begin(), msg.end(), '\n'), msg.end());
1578 0 : cm_msg(MERROR, "Pgsql::Connect", "PQsetdbLogin() to host [%s], port %s, unix socket [%s], database [%s], user [%s], password [%s]: error (%s)", host_name.c_str(), tcp_port.c_str(), unix_socket.c_str(), db_name.c_str(), user_name.c_str(), "xxx", msg.c_str());
1579 0 : Disconnect();
1580 0 : return DB_FILE_ERROR;
1581 0 : }
1582 :
1583 : int status;
1584 :
1585 0 : if (fDebug) {
1586 0 : cm_msg(MINFO, "Pgsql::Connect", "Connected to a PostgreSQL database on host [%s], port %s, unix socket [%s], database [%s], user [%s], password [%s], buffer %zu", host_name.c_str(), tcp_port.c_str(), unix_socket.c_str(), db_name.c_str(), user_name.c_str(), "xxx", fMaxDisconnected);
1587 0 : cm_msg_flush_buffer();
1588 : }
1589 :
1590 0 : fIsConnected = true;
1591 :
1592 0 : int count = 0;
1593 0 : while (fDisconnectedBuffer.size() > 0) {
1594 0 : status = Exec("(flush)", fDisconnectedBuffer.front().c_str());
1595 0 : if (status != DB_SUCCESS) {
1596 0 : return status;
1597 : }
1598 0 : fDisconnectedBuffer.pop_front();
1599 0 : count++;
1600 : }
1601 :
1602 0 : if (count > 0) {
1603 0 : cm_msg(MINFO, "Pgsql::Connect", "Saved %d, lost %d history events accumulated while disconnected from the database", count, fDisconnectedLost);
1604 0 : cm_msg_flush_buffer();
1605 : }
1606 :
1607 0 : assert(fDisconnectedBuffer.size() == 0);
1608 0 : fDisconnectedLost = 0;
1609 :
1610 0 : if (fDownsample) {
1611 0 : status = Prepare("pg_extensions", "select extname from pg_extension where extname = 'timescaledb';");
1612 :
1613 0 : if (status != DB_SUCCESS || PQntuples(fResult) == 0) {
1614 0 : cm_msg(MERROR, "Pgsql::Connect", "TimescaleDB extension not installed");
1615 0 : return DB_FILE_ERROR;
1616 : }
1617 0 : Finalize();
1618 :
1619 0 : status = Prepare("pg_extensions", "select extname from pg_extension where extname = 'timescaledb_toolkit';");
1620 :
1621 0 : if (status != DB_SUCCESS || PQntuples(fResult) == 0) {
1622 0 : cm_msg(MERROR, "Pgsql::Connect", "TimescaleDB_toolkit extension not installed");
1623 0 : return DB_FILE_ERROR;
1624 : }
1625 0 : Finalize();
1626 :
1627 0 : cm_msg(MINFO, "Pgsql::Connect", "TimescaleDB extensions found - downsampling enabled");
1628 : }
1629 :
1630 0 : return DB_SUCCESS;
1631 0 : }
1632 :
1633 0 : int Pgsql::Disconnect()
1634 : {
1635 0 : if (fPgsql)
1636 0 : PQfinish(fPgsql);
1637 :
1638 0 : fPgsql = NULL;
1639 0 : fRow = -1;
1640 :
1641 0 : fIsConnected = false;
1642 0 : return DB_SUCCESS;
1643 : }
1644 :
1645 0 : bool Pgsql::IsConnected()
1646 : {
1647 0 : return fIsConnected;
1648 : }
1649 :
1650 0 : int Pgsql::OpenTransaction(const char* table_name)
1651 : {
1652 0 : return Exec(table_name, "BEGIN TRANSACTION;");
1653 : }
1654 :
1655 0 : int Pgsql::CommitTransaction(const char* table_name)
1656 : {
1657 0 : return Exec(table_name, "COMMIT;");
1658 : }
1659 :
1660 0 : int Pgsql::RollbackTransaction(const char* table_name)
1661 : {
1662 0 : return Exec(table_name, "ROLLBACK;");
1663 : }
1664 :
1665 0 : int Pgsql::ListTables(std::vector<std::string> *plist)
1666 : {
1667 0 : if (!fIsConnected)
1668 0 : return DB_FILE_ERROR;
1669 :
1670 0 : if (fDebug)
1671 0 : printf("Pgsql::ListTables!\n");
1672 :
1673 0 : int status = Prepare("pg_tables", "select tablename from pg_tables where schemaname = 'public';");
1674 :
1675 0 : if (status != DB_SUCCESS) {
1676 0 : cm_msg(MERROR, "Pgsql::ListTables", "error %s (%s)", PQresStatus(PQresultStatus(fResult)), PQresultErrorMessage(fResult));
1677 0 : return DB_FILE_ERROR;
1678 : }
1679 :
1680 : while (1) {
1681 0 : if (Step() != DB_SUCCESS)
1682 0 : break;
1683 0 : std::string tn = GetText(0);
1684 0 : plist->push_back(tn);
1685 0 : };
1686 :
1687 0 : Finalize();
1688 :
1689 0 : return DB_SUCCESS;
1690 : }
1691 :
1692 0 : int Pgsql::ListColumns(const char* table_name, std::vector<std::string> *plist)
1693 : {
1694 0 : if (!fIsConnected)
1695 0 : return DB_FILE_ERROR;
1696 :
1697 0 : if (fDebug)
1698 0 : printf("Pgsql::ListColumns for table \'%s\'\n", table_name);
1699 :
1700 0 : std::string cmd;
1701 0 : cmd += "SELECT column_name, data_type FROM information_schema.columns WHERE table_name = ";
1702 0 : cmd += QuoteString(table_name);
1703 0 : cmd += ";";
1704 :
1705 0 : int status = Prepare(table_name, cmd.c_str());
1706 0 : if (status != DB_SUCCESS)
1707 0 : return status;
1708 :
1709 0 : fNumFields = PQnfields(fResult);
1710 :
1711 : while (1) {
1712 0 : if (Step() != DB_SUCCESS)
1713 0 : break;
1714 0 : std::string cn = GetText(0);
1715 0 : std::string ct = GetText(1);
1716 0 : plist->push_back(cn);
1717 0 : plist->push_back(ct);
1718 0 : };
1719 :
1720 0 : Finalize();
1721 :
1722 0 : return DB_SUCCESS;
1723 0 : }
1724 :
1725 0 : int Pgsql::Exec(const char* table_name, const char* sql)
1726 : {
1727 0 : if (fDebug)
1728 0 : printf("Pgsql::Exec(%s, %s)\n", table_name, sql);
1729 :
1730 0 : if (!fPgsql)
1731 0 : return DB_FILE_ERROR;
1732 :
1733 0 : assert(fPgsql);
1734 0 : assert(fRow == -1);
1735 :
1736 0 : fResult = PQexec(fPgsql, sql);
1737 0 : ExecStatusType err = PQresultStatus(fResult);
1738 0 : if(err != PGRES_TUPLES_OK) {
1739 0 : if(err == PGRES_FATAL_ERROR) {
1740 : // handle fatal error
1741 0 : if(strstr(PQresultErrorMessage(fResult), "already exists"))
1742 0 : return DB_KEY_EXIST;
1743 0 : else return DB_FILE_ERROR;
1744 : }
1745 :
1746 0 : if(PQstatus(fPgsql) == CONNECTION_BAD) {
1747 0 : Disconnect();
1748 0 : return ExecDisconnected(table_name, sql);
1749 : }
1750 : }
1751 :
1752 0 : return DB_SUCCESS;
1753 : }
1754 :
1755 0 : int Pgsql::ExecDisconnected(const char* table_name, const char* sql)
1756 : {
1757 0 : if (fDebug)
1758 0 : printf("Pgsql::ExecDisconnected(%s, %s)\n", table_name, sql);
1759 :
1760 0 : if (fDisconnectedBuffer.size() < fMaxDisconnected) {
1761 0 : fDisconnectedBuffer.push_back(sql);
1762 0 : if (fDisconnectedBuffer.size() >= fMaxDisconnected) {
1763 0 : cm_msg(MERROR, "Pgsql::ExecDisconnected", "Error: Disconnected database buffer overflow, size %zu, subsequent events are lost", fDisconnectedBuffer.size());
1764 : }
1765 : } else {
1766 0 : fDisconnectedLost++;
1767 : }
1768 :
1769 0 : time_t now = time(NULL);
1770 :
1771 0 : if (fNextReconnect == 0 || now >= fNextReconnect) {
1772 0 : int status = Connect(fConnectString.c_str());
1773 0 : if (status == DB_SUCCESS) {
1774 0 : fNextReconnect = 0;
1775 0 : fNextReconnectDelaySec = 0;
1776 : } else {
1777 0 : if (fNextReconnectDelaySec == 0) {
1778 0 : fNextReconnectDelaySec = 5;
1779 0 : } else if (fNextReconnectDelaySec < 10*60) {
1780 0 : fNextReconnectDelaySec *= 2;
1781 : }
1782 0 : if (fDebug) {
1783 0 : cm_msg(MINFO, "Pgsql::ExecDisconnected", "Next reconnect attempt in %d sec, history events buffered %zu, lost %d", fNextReconnectDelaySec, fDisconnectedBuffer.size(), fDisconnectedLost);
1784 0 : cm_msg_flush_buffer();
1785 : }
1786 0 : fNextReconnect = now + fNextReconnectDelaySec;
1787 : }
1788 : }
1789 :
1790 0 : return DB_SUCCESS;
1791 : }
1792 :
1793 0 : int Pgsql::Prepare(const char* table_name, const char* sql)
1794 : {
1795 0 : if (fDebug)
1796 0 : printf("Pgsql::Prepare(%s, %s)\n", table_name, sql);
1797 :
1798 0 : if (!fPgsql)
1799 0 : return DB_FILE_ERROR;
1800 :
1801 0 : assert(fPgsql);
1802 : //assert(fResult==NULL);
1803 0 : assert(fRow == -1);
1804 :
1805 0 : fResult = PQexec(fPgsql, sql);
1806 0 : if (PQstatus(fPgsql) == CONNECTION_BAD) {
1807 : // lost connection to server
1808 0 : int status = Connect(fConnectString.c_str());
1809 0 : if (status == DB_SUCCESS) {
1810 : // Retry after reconnecting
1811 0 : fResult = PQexec(fPgsql, sql);
1812 : } else {
1813 0 : cm_msg(MERROR, "Pgsql::Prepare", "PQexec(%s) PostgreSQL server has gone away, and couldn't reconnect - %d", sql, status);
1814 0 : return DB_FILE_ERROR;
1815 : }
1816 0 : if (status) {
1817 0 : cm_msg(MERROR, "Pgsql::Prepare", "PQexec(%s) error %s", sql, PQresStatus(PQresultStatus(fResult)));
1818 0 : return DB_FILE_ERROR;
1819 : }
1820 0 : cm_msg(MINFO, "Pgsql::Prepare", "Reconnected to PostgreSQL after long inactivity.");
1821 : }
1822 :
1823 0 : fNumFields = PQnfields(fResult);
1824 :
1825 0 : return DB_SUCCESS;
1826 : }
1827 :
1828 0 : std::string Pgsql::BuildDownsampleQuery(const time_t start_time, const time_t end_time, const int npoints,
1829 : const char* table_name, const char* column_name)
1830 : {
1831 0 : std::string cmd;
1832 0 : cmd += "SELECT extract(epoch from time::TIMESTAMPTZ) as _i_time, value ";
1833 :
1834 0 : cmd += " FROM unnest(( SELECT lttb";
1835 0 : cmd += "(_t_time, ";
1836 0 : cmd += column_name;
1837 0 : cmd += ", ";
1838 0 : cmd += std::to_string(npoints);
1839 0 : cmd += ") ";
1840 0 : cmd += "FROM ";
1841 0 : cmd += QuoteId(table_name);
1842 0 : cmd += " WHERE _t_time BETWEEN ";
1843 0 : cmd += "to_timestamp(";
1844 0 : cmd += TimeToString(start_time);
1845 0 : cmd += ") AND to_timestamp(";
1846 0 : cmd += TimeToString(end_time);
1847 0 : cmd += ") )) ORDER BY time;";
1848 :
1849 0 : return cmd;
1850 0 : }
1851 :
1852 0 : int Pgsql::Step()
1853 : {
1854 0 : assert(fPgsql);
1855 0 : assert(fResult);
1856 :
1857 0 : fRow++;
1858 :
1859 0 : if (fRow == PQntuples(fResult))
1860 0 : return DB_NO_MORE_SUBKEYS;
1861 :
1862 0 : return DB_SUCCESS;
1863 : }
1864 :
1865 0 : const char* Pgsql::GetText(int column)
1866 : {
1867 0 : assert(fPgsql);
1868 0 : assert(fResult);
1869 0 : assert(fNumFields > 0);
1870 0 : assert(column >= 0);
1871 0 : assert(column < fNumFields);
1872 :
1873 0 : return PQgetvalue(fResult, fRow, column);
1874 : }
1875 :
1876 0 : double Pgsql::GetDouble(int column)
1877 : {
1878 0 : return atof(GetText(column));
1879 : }
1880 :
1881 0 : time_t Pgsql::GetTime(int column)
1882 : {
1883 0 : return strtoul(GetText(column), NULL, 0);
1884 : }
1885 :
1886 0 : int Pgsql::Finalize()
1887 : {
1888 0 : assert(fPgsql);
1889 0 : assert(fResult);
1890 :
1891 0 : fRow = -1;
1892 0 : fNumFields = 0;
1893 :
1894 0 : return DB_SUCCESS;
1895 : }
1896 :
1897 0 : const char* Pgsql::ColumnType(int midas_tid)
1898 : {
1899 0 : assert(midas_tid>=0);
1900 0 : assert(midas_tid<TID_LAST);
1901 0 : return sql_type_pgsql[midas_tid];
1902 : }
1903 :
1904 0 : bool Pgsql::TypesCompatible(int midas_tid, const char* sql_type)
1905 : {
1906 : if (/* DISABLES CODE */ (0))
1907 : printf("compare types midas \'%s\'=\'%s\' and sql \'%s\'\n", rpc_tid_name(midas_tid), ColumnType(midas_tid), sql_type);
1908 :
1909 : //if (sql2midasType_mysql(sql_type) == midas_tid)
1910 : // return true;
1911 :
1912 0 : if (strcasecmp(ColumnType(midas_tid), sql_type) == 0)
1913 0 : return true;
1914 :
1915 : // permit writing FLOAT into DOUBLE
1916 0 : if (midas_tid==TID_FLOAT && strcmp(sql_type, "double precision")==0)
1917 0 : return true;
1918 :
1919 : // T2K quirk!
1920 : // permit writing BYTE into signed tinyint
1921 0 : if (midas_tid==TID_BYTE && strcmp(sql_type, "integer")==0)
1922 0 : return true;
1923 :
1924 : // T2K quirk!
1925 : // permit writing WORD into signed tinyint
1926 0 : if (midas_tid==TID_WORD && strcmp(sql_type, "integer")==0)
1927 0 : return true;
1928 :
1929 : if (/* DISABLES CODE */ (0))
1930 : printf("type mismatch!\n");
1931 :
1932 0 : return false;
1933 : }
1934 :
1935 0 : std::string Pgsql::QuoteId(const char* s)
1936 : {
1937 0 : std::string q;
1938 0 : q += '"';
1939 0 : q += s;
1940 0 : q += '"';
1941 0 : return q;
1942 0 : }
1943 :
1944 0 : std::string Pgsql::QuoteString(const char* s)
1945 : {
1946 0 : std::string q;
1947 0 : q += '\'';
1948 0 : q += s;
1949 0 : q += '\'';
1950 0 : return q;
1951 0 : }
1952 :
1953 : #endif // HAVE_PGSQL
1954 :
1955 : #ifdef HAVE_SQLITE
1956 :
1957 : ////////////////////////////////////////
1958 : // SQLITE database access //
1959 : ////////////////////////////////////////
1960 :
1961 : #include <sqlite3.h>
1962 :
1963 : typedef std::map<std::string, sqlite3*> DbMap;
1964 :
1965 : class Sqlite: public SqlBase
1966 : {
1967 : public:
1968 : std::string fPath;
1969 :
1970 : DbMap fMap;
1971 :
1972 : // temporary storage of query data
1973 : sqlite3* fTempDB;
1974 : sqlite3_stmt* fTempStmt;
1975 :
1976 : Sqlite(); // ctor
1977 : ~Sqlite(); // dtor
1978 :
1979 : int Connect(const char* path);
1980 : int Disconnect();
1981 : bool IsConnected();
1982 :
1983 : int ConnectTable(const char* table_name);
1984 : sqlite3* GetTable(const char* table_name);
1985 :
1986 : int ListTables(std::vector<std::string> *plist);
1987 : int ListColumns(const char* table_name, std::vector<std::string> *plist);
1988 :
1989 : int Exec(const char* table_name, const char* sql);
1990 : int ExecDisconnected(const char* table_name, const char* sql);
1991 :
1992 : int Prepare(const char* table_name, const char* sql);
1993 : int Step();
1994 : const char* GetText(int column);
1995 : time_t GetTime(int column);
1996 : double GetDouble(int column);
1997 : int Finalize();
1998 :
1999 : int OpenTransaction(const char* table_name);
2000 : int CommitTransaction(const char* table_name);
2001 : int RollbackTransaction(const char* table_name);
2002 :
2003 : const char* ColumnType(int midas_tid);
2004 : bool TypesCompatible(int midas_tid, const char* sql_type);
2005 :
2006 : std::string QuoteId(const char* s);
2007 : std::string QuoteString(const char* s);
2008 : };
2009 :
2010 0 : std::string Sqlite::QuoteId(const char* s)
2011 : {
2012 0 : std::string q;
2013 0 : q += "\"";
2014 0 : q += s;
2015 0 : q += "\"";
2016 0 : return q;
2017 0 : }
2018 :
2019 0 : std::string Sqlite::QuoteString(const char* s)
2020 : {
2021 0 : std::string q;
2022 0 : q += "\'";
2023 0 : q += s;
2024 0 : q += "\'";
2025 0 : return q;
2026 0 : }
2027 :
2028 0 : const char* Sqlite::ColumnType(int midas_tid)
2029 : {
2030 0 : assert(midas_tid>=0);
2031 0 : assert(midas_tid<TID_LAST);
2032 0 : return sql_type_sqlite[midas_tid];
2033 : }
2034 :
2035 0 : bool Sqlite::TypesCompatible(int midas_tid, const char* sql_type)
2036 : {
2037 : if (0)
2038 : printf("compare types midas \'%s\'=\'%s\' and sql \'%s\'\n", rpc_tid_name(midas_tid), ColumnType(midas_tid), sql_type);
2039 :
2040 : //if (sql2midasType_sqlite(sql_type) == midas_tid)
2041 : // return true;
2042 :
2043 0 : if (strcasecmp(ColumnType(midas_tid), sql_type) == 0)
2044 0 : return true;
2045 :
2046 : // permit writing FLOAT into DOUBLE
2047 0 : if (midas_tid==TID_FLOAT && strcasecmp(sql_type, "double")==0)
2048 0 : return true;
2049 :
2050 0 : return false;
2051 : }
2052 :
2053 0 : const char* Sqlite::GetText(int column)
2054 : {
2055 0 : return (const char*)sqlite3_column_text(fTempStmt, column);
2056 : }
2057 :
2058 0 : time_t Sqlite::GetTime(int column)
2059 : {
2060 0 : return sqlite3_column_int64(fTempStmt, column);
2061 : }
2062 :
2063 0 : double Sqlite::GetDouble(int column)
2064 : {
2065 0 : return sqlite3_column_double(fTempStmt, column);
2066 : }
2067 :
2068 0 : Sqlite::Sqlite() // ctor
2069 : {
2070 0 : fIsConnected = false;
2071 0 : fTempDB = NULL;
2072 0 : fTempStmt = NULL;
2073 0 : fDebug = 0;
2074 0 : }
2075 :
2076 0 : Sqlite::~Sqlite() // dtor
2077 : {
2078 0 : Disconnect();
2079 0 : }
2080 :
2081 0 : const char* xsqlite3_errstr(sqlite3* db, int errcode)
2082 : {
2083 : //return sqlite3_errstr(errcode);
2084 0 : return sqlite3_errmsg(db);
2085 : }
2086 :
2087 0 : int Sqlite::ConnectTable(const char* table_name)
2088 : {
2089 0 : std::string fname = fPath + "mh_" + table_name + ".sqlite3";
2090 :
2091 0 : sqlite3* db = NULL;
2092 :
2093 0 : int status = sqlite3_open(fname.c_str(), &db);
2094 :
2095 0 : if (status != SQLITE_OK) {
2096 0 : cm_msg(MERROR, "Sqlite::Connect", "Table %s: sqlite3_open(%s) error %d (%s)", table_name, fname.c_str(), status, xsqlite3_errstr(db, status));
2097 0 : sqlite3_close(db);
2098 0 : db = NULL;
2099 0 : return DB_FILE_ERROR;
2100 : }
2101 :
2102 : #if SQLITE_VERSION_NUMBER >= 3006020
2103 0 : status = sqlite3_extended_result_codes(db, 1);
2104 0 : if (status != SQLITE_OK) {
2105 0 : cm_msg(MERROR, "Sqlite::Connect", "Table %s: sqlite3_extended_result_codes(1) error %d (%s)", table_name, status, xsqlite3_errstr(db, status));
2106 : }
2107 : #else
2108 : #warning Missing sqlite3_extended_result_codes()!
2109 : #endif
2110 :
2111 0 : fMap[table_name] = db;
2112 :
2113 0 : Exec(table_name, "PRAGMA journal_mode=persist;");
2114 0 : Exec(table_name, "PRAGMA synchronous=normal;");
2115 : //Exec(table_name, "PRAGMA synchronous=off;");
2116 0 : Exec(table_name, "PRAGMA journal_size_limit=-1;");
2117 :
2118 : if (0) {
2119 : Exec(table_name, "PRAGMA legacy_file_format;");
2120 : Exec(table_name, "PRAGMA synchronous;");
2121 : Exec(table_name, "PRAGMA journal_mode;");
2122 : Exec(table_name, "PRAGMA journal_size_limit;");
2123 : }
2124 :
2125 : #ifdef SQLITE_LIMIT_COLUMN
2126 : if (0) {
2127 : int max_columns = sqlite3_limit(db, SQLITE_LIMIT_COLUMN, -1);
2128 : printf("Sqlite::Connect: SQLITE_LIMIT_COLUMN=%d\n", max_columns);
2129 : }
2130 : #endif
2131 :
2132 0 : if (fDebug)
2133 0 : cm_msg(MINFO, "Sqlite::Connect", "Table %s: connected to Sqlite file \'%s\'", table_name, fname.c_str());
2134 :
2135 0 : return DB_SUCCESS;
2136 0 : }
2137 :
2138 0 : sqlite3* Sqlite::GetTable(const char* table_name)
2139 : {
2140 0 : sqlite3* db = fMap[table_name];
2141 :
2142 0 : if (db)
2143 0 : return db;
2144 :
2145 0 : int status = ConnectTable(table_name);
2146 0 : if (status != DB_SUCCESS)
2147 0 : return NULL;
2148 :
2149 0 : return fMap[table_name];
2150 : }
2151 :
2152 0 : int Sqlite::Connect(const char* path)
2153 : {
2154 0 : if (fIsConnected)
2155 0 : Disconnect();
2156 :
2157 0 : fPath = path;
2158 :
2159 : // add trailing '/'
2160 0 : if (fPath.length() > 0) {
2161 0 : if (fPath[fPath.length()-1] != DIR_SEPARATOR)
2162 0 : fPath += DIR_SEPARATOR_STR;
2163 : }
2164 :
2165 0 : if (fDebug)
2166 0 : cm_msg(MINFO, "Sqlite::Connect", "Connected to Sqlite database in \'%s\'", fPath.c_str());
2167 :
2168 0 : fIsConnected = true;
2169 :
2170 0 : return DB_SUCCESS;
2171 : }
2172 :
2173 0 : int Sqlite::Disconnect()
2174 : {
2175 0 : if (!fIsConnected)
2176 0 : return DB_SUCCESS;
2177 :
2178 0 : for (DbMap::iterator iter = fMap.begin(); iter != fMap.end(); ++iter) {
2179 0 : const char* table_name = iter->first.c_str();
2180 0 : sqlite3* db = iter->second;
2181 0 : int status = sqlite3_close(db);
2182 0 : if (status != SQLITE_OK) {
2183 0 : cm_msg(MERROR, "Sqlite::Disconnect", "sqlite3_close(%s) error %d (%s)", table_name, status, xsqlite3_errstr(db, status));
2184 : }
2185 : }
2186 :
2187 0 : fMap.clear();
2188 :
2189 0 : fIsConnected = false;
2190 :
2191 0 : return DB_SUCCESS;
2192 : }
2193 :
2194 0 : bool Sqlite::IsConnected()
2195 : {
2196 0 : return fIsConnected;
2197 : }
2198 :
2199 0 : int Sqlite::OpenTransaction(const char* table_name)
2200 : {
2201 0 : int status = Exec(table_name, "BEGIN TRANSACTION");
2202 0 : return status;
2203 : }
2204 :
2205 0 : int Sqlite::CommitTransaction(const char* table_name)
2206 : {
2207 0 : int status = Exec(table_name, "COMMIT TRANSACTION");
2208 0 : return status;
2209 : }
2210 :
2211 0 : int Sqlite::RollbackTransaction(const char* table_name)
2212 : {
2213 0 : int status = Exec(table_name, "ROLLBACK TRANSACTION");
2214 0 : return status;
2215 : }
2216 :
2217 0 : int Sqlite::Prepare(const char* table_name, const char* sql)
2218 : {
2219 0 : sqlite3* db = GetTable(table_name);
2220 0 : if (!db)
2221 0 : return DB_FILE_ERROR;
2222 :
2223 0 : if (fDebug)
2224 0 : printf("Sqlite::Prepare(%s, %s)\n", table_name, sql);
2225 :
2226 0 : assert(fTempDB==NULL);
2227 0 : fTempDB = db;
2228 :
2229 : #if SQLITE_VERSION_NUMBER >= 3006020
2230 0 : int status = sqlite3_prepare_v2(db, sql, strlen(sql), &fTempStmt, NULL);
2231 : #else
2232 : #warning Missing sqlite3_prepare_v2()!
2233 : int status = sqlite3_prepare(db, sql, strlen(sql), &fTempStmt, NULL);
2234 : #endif
2235 :
2236 0 : if (status == SQLITE_OK)
2237 0 : return DB_SUCCESS;
2238 :
2239 0 : std::string sqlstring = sql;
2240 0 : cm_msg(MERROR, "Sqlite::Prepare", "Table %s: sqlite3_prepare_v2(%s...) error %d (%s)", table_name, sqlstring.substr(0,60).c_str(), status, xsqlite3_errstr(db, status));
2241 :
2242 0 : fTempDB = NULL;
2243 :
2244 0 : return DB_FILE_ERROR;
2245 0 : }
2246 :
2247 0 : int Sqlite::Step()
2248 : {
2249 : if (0 && fDebug)
2250 : printf("Sqlite::Step()\n");
2251 :
2252 0 : assert(fTempDB);
2253 0 : assert(fTempStmt);
2254 :
2255 0 : int status = sqlite3_step(fTempStmt);
2256 :
2257 0 : if (status == SQLITE_DONE)
2258 0 : return DB_NO_MORE_SUBKEYS;
2259 :
2260 0 : if (status == SQLITE_ROW)
2261 0 : return DB_SUCCESS;
2262 :
2263 0 : cm_msg(MERROR, "Sqlite::Step", "sqlite3_step() error %d (%s)", status, xsqlite3_errstr(fTempDB, status));
2264 :
2265 0 : return DB_FILE_ERROR;
2266 : }
2267 :
2268 0 : int Sqlite::Finalize()
2269 : {
2270 : if (0 && fDebug)
2271 : printf("Sqlite::Finalize()\n");
2272 :
2273 0 : assert(fTempDB);
2274 0 : assert(fTempStmt);
2275 :
2276 0 : int status = sqlite3_finalize(fTempStmt);
2277 :
2278 0 : if (status != SQLITE_OK) {
2279 0 : cm_msg(MERROR, "Sqlite::Finalize", "sqlite3_finalize() error %d (%s)", status, xsqlite3_errstr(fTempDB, status));
2280 :
2281 0 : fTempDB = NULL;
2282 0 : fTempStmt = NULL; // FIXME: maybe a memory leak?
2283 0 : return DB_FILE_ERROR;
2284 : }
2285 :
2286 0 : fTempDB = NULL;
2287 0 : fTempStmt = NULL;
2288 :
2289 0 : return DB_SUCCESS;
2290 : }
2291 :
2292 0 : int Sqlite::ListTables(std::vector<std::string> *plist)
2293 : {
2294 0 : if (!fIsConnected)
2295 0 : return DB_FILE_ERROR;
2296 :
2297 0 : if (fDebug)
2298 0 : printf("Sqlite::ListTables at path [%s]\n", fPath.c_str());
2299 :
2300 : int status;
2301 :
2302 0 : const char* cmd = "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;";
2303 :
2304 0 : DIR *dir = opendir(fPath.c_str());
2305 0 : if (!dir) {
2306 0 : cm_msg(MERROR, "Sqlite::ListTables", "Cannot opendir(%s), errno %d (%s)", fPath.c_str(), errno, strerror(errno));
2307 0 : return HS_FILE_ERROR;
2308 : }
2309 :
2310 : while (1) {
2311 0 : const struct dirent* de = readdir(dir);
2312 0 : if (!de)
2313 0 : break;
2314 :
2315 0 : const char* dn = de->d_name;
2316 :
2317 : //if (dn[0]!='m' || dn[1]!='h')
2318 : //continue;
2319 :
2320 : const char* s;
2321 :
2322 0 : s = strstr(dn, "mh_");
2323 0 : if (!s || s!=dn)
2324 0 : continue;
2325 :
2326 0 : s = strstr(dn, ".sqlite3");
2327 0 : if (!s || s[8]!=0)
2328 0 : continue;
2329 :
2330 : char table_name[256];
2331 0 : mstrlcpy(table_name, dn+3, sizeof(table_name));
2332 : // FIXME: skip names like "xxx.sqlite3~" and "xxx.sqlite3-deleted"
2333 0 : char* ss = strstr(table_name, ".sqlite3");
2334 0 : if (!ss)
2335 0 : continue;
2336 0 : *ss = 0;
2337 :
2338 : //printf("dn [%s] tn [%s]\n", dn, table_name);
2339 :
2340 0 : status = Prepare(table_name, cmd);
2341 0 : if (status != DB_SUCCESS)
2342 0 : continue;
2343 :
2344 : while (1) {
2345 0 : status = Step();
2346 0 : if (status != DB_SUCCESS)
2347 0 : break;
2348 :
2349 0 : const char* tn = GetText(0);
2350 : //printf("table [%s]\n", tn);
2351 0 : plist->push_back(tn);
2352 0 : }
2353 :
2354 0 : status = Finalize();
2355 0 : }
2356 :
2357 0 : closedir(dir);
2358 0 : dir = NULL;
2359 :
2360 0 : return DB_SUCCESS;
2361 : }
2362 :
2363 0 : int Sqlite::ListColumns(const char* table, std::vector<std::string> *plist)
2364 : {
2365 0 : if (!fIsConnected)
2366 0 : return DB_FILE_ERROR;
2367 :
2368 0 : if (fDebug)
2369 0 : printf("Sqlite::ListColumns for table \'%s\'\n", table);
2370 :
2371 0 : std::string cmd;
2372 0 : cmd = "PRAGMA table_info(";
2373 0 : cmd += table;
2374 0 : cmd += ");";
2375 :
2376 : int status;
2377 :
2378 0 : status = Prepare(table, cmd.c_str());
2379 0 : if (status != DB_SUCCESS)
2380 0 : return status;
2381 :
2382 : while (1) {
2383 0 : status = Step();
2384 0 : if (status != DB_SUCCESS)
2385 0 : break;
2386 :
2387 0 : const char* colname = GetText(1);
2388 0 : const char* coltype = GetText(2);
2389 : //printf("column [%s] [%s]\n", colname, coltype);
2390 0 : plist->push_back(colname); // column name
2391 0 : plist->push_back(coltype); // column type
2392 0 : }
2393 :
2394 0 : status = Finalize();
2395 :
2396 0 : return DB_SUCCESS;
2397 0 : }
2398 :
2399 : static int callback_debug = 0;
2400 :
2401 0 : static int callback(void *NotUsed, int argc, char **argv, char **azColName){
2402 0 : if (callback_debug) {
2403 0 : printf("history_sqlite::callback---->\n");
2404 0 : for (int i=0; i<argc; i++){
2405 0 : printf("history_sqlite::callback[%d] %s = %s\n", i, azColName[i], argv[i] ? argv[i] : "NULL");
2406 : }
2407 : }
2408 0 : return 0;
2409 : }
2410 :
2411 0 : int Sqlite::Exec(const char* table_name, const char* sql)
2412 : {
2413 : // return values:
2414 : // DB_SUCCESS
2415 : // DB_FILE_ERROR: not connected
2416 : // DB_KEY_EXIST: "table already exists"
2417 :
2418 0 : if (!fIsConnected)
2419 0 : return DB_FILE_ERROR;
2420 :
2421 0 : sqlite3* db = GetTable(table_name);
2422 0 : if (!db)
2423 0 : return DB_FILE_ERROR;
2424 :
2425 0 : if (fDebug)
2426 0 : printf("Sqlite::Exec(%s, %s)\n", table_name, sql);
2427 :
2428 : int status;
2429 :
2430 0 : callback_debug = fDebug;
2431 0 : char* errmsg = NULL;
2432 :
2433 0 : status = sqlite3_exec(db, sql, callback, 0, &errmsg);
2434 0 : if (status != SQLITE_OK) {
2435 0 : if (status == SQLITE_ERROR && strstr(errmsg, "duplicate column name"))
2436 0 : return DB_KEY_EXIST;
2437 0 : if (status == SQLITE_ERROR && strstr(errmsg, "already exists"))
2438 0 : return DB_KEY_EXIST;
2439 0 : std::string sqlstring = sql;
2440 0 : cm_msg(MERROR, "Sqlite::Exec", "Table %s: sqlite3_exec(%s...) error %d (%s)", table_name, sqlstring.substr(0,60).c_str(), status, errmsg);
2441 0 : sqlite3_free(errmsg);
2442 0 : return DB_FILE_ERROR;
2443 0 : }
2444 :
2445 0 : return DB_SUCCESS;
2446 : }
2447 :
2448 0 : int Sqlite::ExecDisconnected(const char* table_name, const char* sql)
2449 : {
2450 0 : cm_msg(MERROR, "Sqlite::Exec", "sqlite driver does not support disconnected operations");
2451 0 : return DB_FILE_ERROR;
2452 : }
2453 :
2454 : #endif // HAVE_SQLITE
2455 :
2456 : ////////////////////////////////////
2457 : // Methods of HsFileSchema //
2458 : ////////////////////////////////////
2459 :
2460 0 : int HsFileSchema::write_event(const time_t t, const char* data, const size_t data_size)
2461 : {
2462 0 : HsFileSchema* s = this;
2463 :
2464 0 : assert(s->fVariables.size() == s->fOffsets.size());
2465 :
2466 0 : if (s->fWriterFd < 0) {
2467 0 : s->fWriterFd = open(s->fFileName.c_str(), O_RDWR);
2468 0 : if (s->fWriterFd < 0) {
2469 0 : cm_msg(MERROR, "FileHistory::write_event", "Cannot write to \'%s\', open() errno %d (%s)", s->fFileName.c_str(), errno, strerror(errno));
2470 0 : return HS_FILE_ERROR;
2471 : }
2472 :
2473 : //static_assert((sizeof(int)==3),"here!");
2474 : //static_assert((sizeof(off64_t)==7),"here!");
2475 : //static_assert((sizeof(size_t)==3),"here!");
2476 : //static_assert((sizeof(intmax_t)==3),"here!");
2477 :
2478 : //off64_t zzz = 1;
2479 : //off64_t xxx = zzz<<60;
2480 : //int yyy = xxx;
2481 : //printf("%d", yyy);
2482 :
2483 0 : off64_t file_size = ::lseek64(s->fWriterFd, 0, SEEK_END);
2484 :
2485 0 : if (file_size < 0) {
2486 0 : cm_msg(MERROR, "FileHistory::write_event", "Cannot read file size of \'%s\', lseek64(SEEK_END) errno %d (%s)", s->fFileName.c_str(), errno, strerror(errno));
2487 0 : return HS_FILE_ERROR;
2488 : }
2489 :
2490 0 : off64_t nrec = 0;
2491 :
2492 0 : if (file_size >= s->fDataOffset) {
2493 0 : nrec = (file_size - s->fDataOffset)/s->fRecordSize;
2494 : }
2495 :
2496 0 : if (nrec < 0)
2497 0 : nrec = 0;
2498 :
2499 0 : off64_t data_end = s->fDataOffset + nrec*s->fRecordSize;
2500 :
2501 : //printf("write_event: file_size %jd, nrec %jd, data_end %jd\n", (intmax_t)file_size, (intmax_t)nrec, (intmax_t)data_end);
2502 :
2503 0 : if (data_end != file_size) {
2504 0 : if (nrec > 0)
2505 0 : cm_msg(MERROR, "FileHistory::write_event", "File \'%s\' may be truncated, data offset %jd, record size %zu, file size: %jd, should be %jd, making it so", s->fFileName.c_str(), (intmax_t)s->fDataOffset, s->fRecordSize, (intmax_t)file_size, (intmax_t)data_end);
2506 :
2507 0 : off64_t status64 = ::lseek64(s->fWriterFd, data_end, SEEK_SET);
2508 :
2509 0 : if (status64 < 0) {
2510 0 : cm_msg(MERROR, "FileHistory::write_event", "Cannot seek \'%s\' to offset %jd, lseek64() errno %d (%s)", s->fFileName.c_str(), (intmax_t)data_end, errno, strerror(errno));
2511 0 : return HS_FILE_ERROR;
2512 : }
2513 :
2514 0 : int status = ::ftruncate64(s->fWriterFd, data_end);
2515 :
2516 0 : if (status < 0) {
2517 0 : cm_msg(MERROR, "FileHistory::write_event", "Cannot truncate \'%s\' to size %jd, ftruncate64() errno %d (%s)", s->fFileName.c_str(), (intmax_t)data_end, errno, strerror(errno));
2518 0 : return HS_FILE_ERROR;
2519 : }
2520 : }
2521 :
2522 0 : s->fFileSizeInitial = data_end;
2523 0 : s->fFileSize = data_end;
2524 : }
2525 :
2526 0 : size_t expected_size = s->fRecordSize - 4;
2527 :
2528 : // sanity check: record_size and n_bytes are computed from the byte counts in the file header
2529 0 : assert(expected_size == s->fNumBytes);
2530 :
2531 0 : if (s->fLastSize == 0)
2532 0 : s->fLastSize = expected_size;
2533 :
2534 0 : if (data_size != s->fLastSize) {
2535 0 : cm_msg(MERROR, "FileHistory::write_event", "Event \'%s\' data size mismatch, expected %zu bytes, got %zu bytes, previously %zu bytes", s->fEventName.c_str(), expected_size, data_size, s->fLastSize);
2536 : //printf("schema:\n");
2537 : //s->print();
2538 :
2539 0 : if (data_size < expected_size)
2540 0 : return HS_FILE_ERROR;
2541 :
2542 : // truncate for now
2543 : // data_size = expected_size;
2544 0 : s->fLastSize = data_size;
2545 : }
2546 :
2547 0 : size_t size = 4 + expected_size;
2548 :
2549 0 : if (size != s->fRecordBufferSize) {
2550 0 : s->fRecordBuffer = (char*)realloc(s->fRecordBuffer, size);
2551 0 : assert(s->fRecordBuffer != NULL);
2552 0 : s->fRecordBufferSize = size;
2553 : }
2554 :
2555 0 : memcpy(s->fRecordBuffer, &t, 4);
2556 0 : memcpy(s->fRecordBuffer+4, data, expected_size);
2557 :
2558 0 : ssize_t wr = write(s->fWriterFd, s->fRecordBuffer, size);
2559 0 : if ((size_t)wr != size) {
2560 0 : cm_msg(MERROR, "FileHistory::write_event", "Cannot write to \'%s\', write(%zu) returned %zd, errno %d (%s)", s->fFileName.c_str(), size, wr, errno, strerror(errno));
2561 0 : return HS_FILE_ERROR;
2562 : }
2563 :
2564 0 : s->fFileSize += size;
2565 :
2566 : #if 0
2567 : status = write(s->fWriterFd, &t, 4);
2568 : if (status != 4) {
2569 : cm_msg(MERROR, "FileHistory::write_event", "Cannot write to \'%s\', write(timestamp) errno %d (%s)", s->fFileName.c_str(), errno, strerror(errno));
2570 : return HS_FILE_ERROR;
2571 : }
2572 :
2573 : status = write(s->fWriterFd, data, expected_size);
2574 : if (status != expected_size) {
2575 : cm_msg(MERROR, "FileHistory::write_event", "Cannot write to \'%s\', write(%d) errno %d (%s)", s->fFileName.c_str(), data_size, errno, strerror(errno));
2576 : return HS_FILE_ERROR;
2577 : }
2578 : #endif
2579 :
2580 0 : return HS_SUCCESS;
2581 : }
2582 :
2583 0 : int HsFileSchema::close()
2584 : {
2585 0 : if (fWriterFd >= 0) {
2586 0 : ::close(fWriterFd);
2587 0 : fWriterFd = -1;
2588 : }
2589 0 : return HS_SUCCESS;
2590 : }
2591 :
2592 0 : static int ReadRecord(const char* file_name, int fd, off64_t offset, size_t recsize, off64_t irec, char* rec)
2593 : {
2594 0 : off64_t fpos = offset + irec*recsize;
2595 :
2596 0 : off64_t status64 = ::lseek64(fd, fpos, SEEK_SET);
2597 :
2598 0 : if (status64 < 0) {
2599 0 : cm_msg(MERROR, "FileHistory::ReadRecord", "Cannot read \'%s\', lseek64(%jd) errno %d (%s)", file_name, (intmax_t)fpos, errno, strerror(errno));
2600 0 : return -1;
2601 : }
2602 :
2603 0 : ssize_t rd = ::read(fd, rec, recsize);
2604 :
2605 0 : if (rd < 0) {
2606 0 : cm_msg(MERROR, "FileHistory::ReadRecord", "Cannot read \'%s\', read() errno %d (%s)", file_name, errno, strerror(errno));
2607 0 : return -1;
2608 : }
2609 :
2610 0 : if (rd == 0) {
2611 0 : cm_msg(MERROR, "FileHistory::ReadRecord", "Cannot read \'%s\', unexpected end of file on read()", file_name);
2612 0 : return -1;
2613 : }
2614 :
2615 0 : if ((size_t)rd != recsize) {
2616 0 : cm_msg(MERROR, "FileHistory::ReadRecord", "Cannot read \'%s\', short read() returned %zd instead of %zu bytes", file_name, rd, recsize);
2617 0 : return -1;
2618 : }
2619 :
2620 0 : return HS_SUCCESS;
2621 : }
2622 :
2623 0 : static int FindTime(const char* file_name, int fd, off64_t offset, size_t recsize, off64_t nrec, time_t timestamp, off64_t* i1p, time_t* t1p, off64_t* i2p, time_t* t2p, time_t* tstart, time_t* tend, int debug)
2624 : {
2625 : //
2626 : // purpose: find location time timestamp inside given file.
2627 : // uses binary search
2628 : // returns:
2629 : // tstart, tend - time of first and last data in a file
2630 : // i1p,t1p - data just before timestamp, used as "last_written"
2631 : // i2p,t2p - data at timestamp or after timestamp, used as starting point to read data from file
2632 : // assertions:
2633 : // tstart <= t1p < t2p <= tend
2634 : // i1p+1==i2p
2635 : // t1p < timestamp <= t2p
2636 : //
2637 : // special cases:
2638 : // 1) timestamp <= tstart - all data is in the future, return i1p==-1, t1p==-1, i2p==0, t2p==tstart
2639 : // 2) tend < timestamp - all the data is in the past, return i1p = nrec-1, t1p = tend, i2p = nrec, t2p = 0;
2640 : // 3) nrec == 1 only one record in this file and it is older than the timestamp (tstart == tend < timestamp)
2641 : //
2642 :
2643 : int status;
2644 0 : char* buf = new char[recsize];
2645 :
2646 0 : assert(nrec > 0);
2647 :
2648 0 : off64_t rec1 = 0;
2649 0 : off64_t rec2 = nrec-1;
2650 :
2651 0 : status = ReadRecord(file_name, fd, offset, recsize, rec1, buf);
2652 0 : if (status != HS_SUCCESS) {
2653 0 : delete[] buf;
2654 0 : return HS_FILE_ERROR;
2655 : }
2656 :
2657 0 : time_t t1 = *(DWORD*)buf;
2658 :
2659 0 : *tstart = t1;
2660 :
2661 : // timestamp is older than any data in this file
2662 0 : if (timestamp <= t1) {
2663 0 : *i1p = -1;
2664 0 : *t1p = 0;
2665 0 : *i2p = 0;
2666 0 : *t2p = t1;
2667 0 : *tend = 0;
2668 0 : delete[] buf;
2669 0 : return HS_SUCCESS;
2670 : }
2671 :
2672 0 : assert(t1 < timestamp);
2673 :
2674 0 : if (nrec == 1) {
2675 0 : *i1p = 0;
2676 0 : *t1p = t1;
2677 0 : *i2p = nrec; // == 1
2678 0 : *t2p = 0;
2679 0 : *tend = t1;
2680 0 : delete[] buf;
2681 0 : return HS_SUCCESS;
2682 : }
2683 :
2684 0 : status = ReadRecord(file_name, fd, offset, recsize, rec2, buf);
2685 0 : if (status != HS_SUCCESS) {
2686 0 : delete[] buf;
2687 0 : return HS_FILE_ERROR;
2688 : }
2689 :
2690 0 : time_t t2 = *(DWORD*)buf;
2691 :
2692 0 : *tend = t2;
2693 :
2694 : // all the data is in the past
2695 0 : if (t2 < timestamp) {
2696 0 : *i1p = rec2;
2697 0 : *t1p = t2;
2698 0 : *i2p = nrec;
2699 0 : *t2p = 0;
2700 0 : delete[] buf;
2701 0 : return HS_SUCCESS;
2702 : }
2703 :
2704 0 : assert(t1 < timestamp);
2705 0 : assert(timestamp <= t2);
2706 :
2707 0 : if (debug)
2708 0 : printf("FindTime: rec %jd..(x)..%jd, time %s..(%s)..%s\n", (intmax_t)rec1, (intmax_t)rec2, TimeToString(t1).c_str(), TimeToString(timestamp).c_str(), TimeToString(t2).c_str());
2709 :
2710 : // implement binary search
2711 :
2712 : do {
2713 0 : off64_t rec = (rec1+rec2)/2;
2714 :
2715 0 : assert(rec >= 0);
2716 0 : assert(rec < nrec);
2717 :
2718 0 : status = ReadRecord(file_name, fd, offset, recsize, rec, buf);
2719 0 : if (status != HS_SUCCESS) {
2720 0 : delete[] buf;
2721 0 : return HS_FILE_ERROR;
2722 : }
2723 :
2724 0 : time_t t = *(DWORD*)buf;
2725 :
2726 0 : if (timestamp <= t) {
2727 0 : if (debug)
2728 0 : printf("FindTime: rec %jd..(x)..%jd..%jd, time %s..(%s)..%s..%s\n", (intmax_t)rec1, (intmax_t)rec, (intmax_t)rec2, TimeToString(t1).c_str(), TimeToString(timestamp).c_str(), TimeToString(t).c_str(), TimeToString(t2).c_str());
2729 :
2730 0 : rec2 = rec;
2731 0 : t2 = t;
2732 : } else {
2733 0 : if (debug)
2734 0 : printf("FindTime: rec %jd..%jd..(x)..%jd, time %s..%s..(%s)..%s\n", (intmax_t)rec1, (intmax_t)rec, (intmax_t)rec2, TimeToString(t1).c_str(), TimeToString(t).c_str(), TimeToString(timestamp).c_str(), TimeToString(t2).c_str());
2735 :
2736 0 : rec1 = rec;
2737 0 : t1 = t;
2738 : }
2739 0 : } while (rec2 - rec1 > 1);
2740 :
2741 0 : assert(rec1+1 == rec2);
2742 0 : assert(t1 < timestamp);
2743 0 : assert(timestamp <= t2);
2744 :
2745 0 : if (debug)
2746 0 : printf("FindTime: rec %jd..(x)..%jd, time %s..(%s)..%s, this is the result.\n", (intmax_t)rec1, (intmax_t)rec2, TimeToString(t1).c_str(), TimeToString(timestamp).c_str(), TimeToString(t2).c_str());
2747 :
2748 0 : *i1p = rec1;
2749 0 : *t1p = t1;
2750 :
2751 0 : *i2p = rec2;
2752 0 : *t2p = t2;
2753 :
2754 0 : delete[] buf;
2755 0 : return HS_SUCCESS;
2756 : }
2757 :
2758 0 : int HsFileSchema::read_last_written(const time_t timestamp,
2759 : const int debug,
2760 : time_t* last_written)
2761 : {
2762 : int status;
2763 0 : HsFileSchema* s = this;
2764 :
2765 0 : if (debug)
2766 0 : printf("FileHistory::read_last_written: file %s, schema time %s..%s, timestamp %s\n", s->fFileName.c_str(), TimeToString(s->fTimeFrom).c_str(), TimeToString(s->fTimeTo).c_str(), TimeToString(timestamp).c_str());
2767 :
2768 0 : int fd = open(s->fFileName.c_str(), O_RDONLY);
2769 0 : if (fd < 0) {
2770 0 : cm_msg(MERROR, "FileHistory::read_last_written", "Cannot read \'%s\', open() errno %d (%s)", s->fFileName.c_str(), errno, strerror(errno));
2771 0 : return HS_FILE_ERROR;
2772 : }
2773 :
2774 0 : off64_t file_size = ::lseek64(fd, 0, SEEK_END);
2775 :
2776 0 : if (file_size < 0) {
2777 0 : cm_msg(MERROR, "FileHistory::read_last_written", "Cannot read file size of \'%s\', lseek64(SEEK_END) errno %d (%s)", s->fFileName.c_str(), errno, strerror(errno));
2778 0 : return HS_FILE_ERROR;
2779 : }
2780 :
2781 0 : if (file_size < s->fDataOffset) {
2782 : // empty file
2783 0 : ::close(fd);
2784 0 : if (last_written)
2785 0 : *last_written = 0;
2786 0 : return HS_SUCCESS;
2787 : }
2788 :
2789 0 : off64_t nrec = (file_size - s->fDataOffset)/s->fRecordSize;
2790 :
2791 : //printf("read_last_written: nrec %jd, file_size %jd, offset %jd, record size %zu\n", (intmax_t)nrec, (intmax_t)file_size, s->fDataOffset, s->fRecordSize);
2792 :
2793 0 : if (nrec < 0)
2794 0 : nrec = 0;
2795 :
2796 0 : if (nrec < 1) {
2797 0 : ::close(fd);
2798 0 : if (last_written)
2799 0 : *last_written = 0;
2800 0 : return HS_SUCCESS;
2801 : }
2802 :
2803 0 : time_t lw = 0;
2804 :
2805 : // read last record to check if desired time is inside or outside of the file
2806 :
2807 : if (1) {
2808 0 : char* buf = new char[s->fRecordSize];
2809 :
2810 0 : status = ReadRecord(s->fFileName.c_str(), fd, s->fDataOffset, s->fRecordSize, nrec - 1, buf);
2811 0 : if (status != HS_SUCCESS) {
2812 0 : delete[] buf;
2813 0 : ::close(fd);
2814 0 : return HS_FILE_ERROR;
2815 : }
2816 :
2817 0 : lw = *(DWORD*)buf;
2818 :
2819 0 : delete[] buf;
2820 : }
2821 :
2822 0 : if (lw >= timestamp) {
2823 0 : off64_t irec = 0;
2824 0 : time_t trec = 0;
2825 0 : off64_t iunused = 0;
2826 0 : time_t tunused = 0;
2827 0 : time_t tstart = 0; // not used
2828 0 : time_t tend = 0; // not used
2829 :
2830 0 : status = FindTime(s->fFileName.c_str(), fd, s->fDataOffset, s->fRecordSize, nrec, timestamp, &irec, &trec, &iunused, &tunused, &tstart, &tend, 0*debug);
2831 0 : if (status != HS_SUCCESS) {
2832 0 : ::close(fd);
2833 0 : return HS_FILE_ERROR;
2834 : }
2835 :
2836 0 : assert(trec < timestamp);
2837 :
2838 0 : lw = trec;
2839 : }
2840 :
2841 0 : if (last_written)
2842 0 : *last_written = lw;
2843 :
2844 0 : if (debug)
2845 0 : printf("FileHistory::read_last_written: file %s, schema time %s..%s, timestamp %s, last_written %s\n", s->fFileName.c_str(), TimeToString(s->fTimeFrom).c_str(), TimeToString(s->fTimeTo).c_str(), TimeToString(timestamp).c_str(), TimeToString(lw).c_str());
2846 :
2847 0 : assert(lw < timestamp);
2848 :
2849 0 : ::close(fd);
2850 :
2851 0 : return HS_SUCCESS;
2852 : }
2853 :
2854 0 : int HsFileSchema::read_data(const time_t start_time,
2855 : const time_t end_time,
2856 : const int num_var, const std::vector<int>& var_schema_index, const int var_index[],
2857 : const int debug,
2858 : std::vector<time_t>& last_time,
2859 : MidasHistoryBufferInterface* buffer[])
2860 : {
2861 0 : HsFileSchema* s = this;
2862 :
2863 0 : if (debug)
2864 0 : printf("FileHistory::read_data: file %s, schema time %s..%s, read time %s..%s, %d vars\n", s->fFileName.c_str(), TimeToString(s->fTimeFrom).c_str(), TimeToString(s->fTimeTo).c_str(), TimeToString(start_time).c_str(), TimeToString(end_time).c_str(), num_var);
2865 :
2866 : //if (1) {
2867 : // printf("Last time: ");
2868 : // for (int i=0; i<num_var; i++) {
2869 : // printf(" %s", TimeToString(last_time[i]).c_str());
2870 : // }
2871 : // printf("\n");
2872 : //}
2873 :
2874 0 : if (debug) {
2875 0 : printf("FileHistory::read_data: file %s map", s->fFileName.c_str());
2876 0 : for (size_t i=0; i<var_schema_index.size(); i++) {
2877 0 : printf(" %2d", var_schema_index[i]);
2878 : }
2879 0 : printf("\n");
2880 : }
2881 :
2882 0 : int fd = ::open(s->fFileName.c_str(), O_RDONLY);
2883 0 : if (fd < 0) {
2884 0 : cm_msg(MERROR, "FileHistory::read_data", "Cannot read \'%s\', open() errno %d (%s)", s->fFileName.c_str(), errno, strerror(errno));
2885 0 : return HS_FILE_ERROR;
2886 : }
2887 :
2888 0 : off64_t file_size = ::lseek64(fd, 0, SEEK_END);
2889 :
2890 0 : if (file_size < 0) {
2891 0 : cm_msg(MERROR, "FileHistory::read_data", "Cannot read file size of \'%s\', lseek64(SEEK_END) errno %d (%s)", s->fFileName.c_str(), errno, strerror(errno));
2892 0 : ::close(fd);
2893 0 : return HS_FILE_ERROR;
2894 : }
2895 :
2896 0 : if (file_size < s->fDataOffset) {
2897 : // empty file
2898 0 : ::close(fd);
2899 0 : return HS_SUCCESS;
2900 : }
2901 :
2902 0 : off64_t nrec = (file_size - s->fDataOffset)/s->fRecordSize;
2903 :
2904 : //printf("read_data: nrec %jd, file_size %jd, offset %jd, record size %zu\n", (intmax_t)nrec, (intmax_t)file_size, s->fDataOffset, s->fRecordSize);
2905 :
2906 0 : if (nrec < 0)
2907 0 : nrec = 0;
2908 :
2909 0 : if (nrec < 1) {
2910 0 : ::close(fd);
2911 0 : return HS_SUCCESS;
2912 : }
2913 :
2914 0 : off64_t iunused = 0;
2915 0 : time_t tunused = 0;
2916 0 : off64_t irec = 0;
2917 0 : time_t trec = 0;
2918 0 : time_t tstart = 0;
2919 0 : time_t tend = 0;
2920 :
2921 0 : int istatus = FindTime(s->fFileName.c_str(), fd, s->fDataOffset, s->fRecordSize, nrec, start_time, &iunused, &tunused, &irec, &trec, &tstart, &tend, 0*debug);
2922 :
2923 0 : if (istatus != HS_SUCCESS) {
2924 0 : ::close(fd);
2925 0 : return HS_FILE_ERROR;
2926 : }
2927 :
2928 0 : if (debug) {
2929 0 : printf("FindTime %d, nrec %jd, (%jd, %s) (%jd, %s), tstart %s, tend %s, want %s\n", istatus, (intmax_t)nrec, (intmax_t)iunused, TimeToString(tunused).c_str(), (intmax_t)irec, TimeToString(trec).c_str(), TimeToString(tstart).c_str(), TimeToString(tend).c_str(), TimeToString(start_time).c_str());
2930 : }
2931 :
2932 0 : if (irec < 0 || irec >= nrec) {
2933 : // all data in this file is older than start_time
2934 :
2935 0 : ::close(fd);
2936 :
2937 0 : if (debug)
2938 0 : printf("FileHistory::read: file %s, schema time %s..%s, read time %s..%s, file time %s..%s, data in this file is too old\n", s->fFileName.c_str(), TimeToString(s->fTimeFrom).c_str(), TimeToString(s->fTimeTo).c_str(), TimeToString(start_time).c_str(), TimeToString(end_time).c_str(), TimeToString(tstart).c_str(), TimeToString(tend).c_str());
2939 :
2940 0 : return HS_SUCCESS;
2941 : }
2942 :
2943 0 : if (tstart < s->fTimeFrom) {
2944 : // data starts before time declared in schema
2945 :
2946 0 : ::close(fd);
2947 :
2948 0 : cm_msg(MERROR, "FileHistory::read_data", "Bad history file \'%s\': timestamp of first data %s is before schema start time %s", s->fFileName.c_str(), TimeToString(tstart).c_str(), TimeToString(s->fTimeFrom).c_str());
2949 :
2950 0 : return HS_FILE_ERROR;
2951 : }
2952 :
2953 0 : if (tend && s->fTimeTo && tend > s->fTimeTo) {
2954 : // data ends after time declared in schema (overlaps with next file)
2955 :
2956 0 : ::close(fd);
2957 :
2958 0 : cm_msg(MERROR, "FileHistory::read_data", "Bad history file \'%s\': timestamp of last data %s is after schema end time %s", s->fFileName.c_str(), TimeToString(tend).c_str(), TimeToString(s->fTimeTo).c_str());
2959 :
2960 0 : return HS_FILE_ERROR;
2961 : }
2962 :
2963 0 : for (int i=0; i<num_var; i++) {
2964 0 : int si = var_schema_index[i];
2965 0 : if (si < 0)
2966 0 : continue;
2967 :
2968 0 : if (trec < last_time[i]) { // protect against duplicate and non-monotonous data
2969 0 : ::close(fd);
2970 :
2971 0 : cm_msg(MERROR, "FileHistory::read_data", "Internal history error at file \'%s\': variable %d data timestamp %s is before last timestamp %s", s->fFileName.c_str(), i, TimeToString(trec).c_str(), TimeToString(last_time[i]).c_str());
2972 :
2973 0 : return HS_FILE_ERROR;
2974 : }
2975 : }
2976 :
2977 0 : int count = 0;
2978 :
2979 0 : off64_t fpos = s->fDataOffset + irec*s->fRecordSize;
2980 :
2981 0 : off64_t xpos = ::lseek64(fd, fpos, SEEK_SET);
2982 :
2983 : //printf("read_data: lseek64() returned %jd, irec %jd, offset %jd, record size %zu, fpos %jd\n", xpos, irec, s->fDataOffset, s->fRecordSize, fpos);
2984 :
2985 0 : if (xpos < 0) {
2986 0 : cm_msg(MERROR, "FileHistory::read_data", "Cannot read \'%s\', lseek64(%jd) errno %d (%s)", s->fFileName.c_str(), (intmax_t)fpos, errno, strerror(errno));
2987 0 : ::close(fd);
2988 0 : return HS_FILE_ERROR;
2989 : }
2990 :
2991 0 : char* buf = new char[s->fRecordSize];
2992 :
2993 0 : off64_t prec = irec;
2994 :
2995 : while (1) {
2996 0 : ssize_t rd = ::read(fd, buf, s->fRecordSize);
2997 :
2998 0 : if (rd < 0) {
2999 0 : cm_msg(MERROR, "FileHistory::read_data", "Cannot read \'%s\', read() errno %d (%s)", s->fFileName.c_str(), errno, strerror(errno));
3000 0 : break;
3001 : }
3002 :
3003 0 : if (rd == 0) {
3004 : // EOF
3005 0 : break;
3006 : }
3007 :
3008 0 : if ((size_t)rd != s->fRecordSize) {
3009 0 : cm_msg(MERROR, "FileHistory::read_data", "Cannot read \'%s\', short read() returned %zd instead of %zu bytes", s->fFileName.c_str(), rd, s->fRecordSize);
3010 0 : break;
3011 : }
3012 :
3013 0 : prec++;
3014 :
3015 0 : bool past_end_of_last_file = (s->fTimeTo == 0) && (prec > nrec);
3016 :
3017 0 : time_t t = *(DWORD*)buf;
3018 :
3019 0 : if (debug > 1)
3020 0 : printf("FileHistory::read: file %s, schema time %s..%s, read time %s..%s, row time %s\n", s->fFileName.c_str(), TimeToString(s->fTimeFrom).c_str(), TimeToString(s->fTimeTo).c_str(), TimeToString(start_time).c_str(), TimeToString(end_time).c_str(), TimeToString(t).c_str());
3021 :
3022 0 : if (t < trec) {
3023 0 : delete[] buf;
3024 0 : ::close(fd);
3025 0 : cm_msg(MERROR, "FileHistory::read_data", "Bad history file \'%s\': record %jd timestamp %s is before start time %s", s->fFileName.c_str(), (intmax_t)(irec + count), TimeToString(t).c_str(), TimeToString(trec).c_str());
3026 0 : return HS_FILE_ERROR;
3027 : }
3028 :
3029 0 : if (tend && (t > tend) && !past_end_of_last_file) {
3030 0 : delete[] buf;
3031 0 : ::close(fd);
3032 0 : cm_msg(MERROR, "FileHistory::read_data", "Bad history file \'%s\': record %jd timestamp %s is after last timestamp %s", s->fFileName.c_str(), (intmax_t)(irec + count), TimeToString(t).c_str(), TimeToString(tend).c_str());
3033 0 : return HS_FILE_ERROR;
3034 : }
3035 :
3036 0 : if (t > end_time)
3037 0 : break;
3038 :
3039 0 : char* data = buf + 4;
3040 :
3041 0 : for (int i=0; i<num_var; i++) {
3042 0 : int si = var_schema_index[i];
3043 0 : if (si < 0)
3044 0 : continue;
3045 :
3046 0 : if (t < last_time[i]) { // protect against duplicate and non-monotonous data
3047 0 : delete[] buf;
3048 0 : ::close(fd);
3049 :
3050 0 : cm_msg(MERROR, "FileHistory::read_data", "Bad history file \'%s\': record %jd timestamp %s is before timestamp %s of variable %d", s->fFileName.c_str(), (intmax_t)(irec + count), TimeToString(t).c_str(), TimeToString(last_time[i]).c_str(), i);
3051 :
3052 0 : return HS_FILE_ERROR;
3053 : }
3054 :
3055 0 : double v = 0;
3056 0 : void* ptr = data + s->fOffsets[si];
3057 :
3058 0 : int ii = var_index[i];
3059 0 : assert(ii >= 0);
3060 0 : assert(ii < s->fVariables[si].n_data);
3061 :
3062 0 : switch (s->fVariables[si].type) {
3063 0 : default:
3064 : // unknown data type
3065 0 : v = 0;
3066 0 : break;
3067 0 : case TID_BYTE:
3068 0 : v = ((unsigned char*)ptr)[ii];
3069 0 : break;
3070 0 : case TID_SBYTE:
3071 0 : v = ((signed char *)ptr)[ii];
3072 0 : break;
3073 0 : case TID_CHAR:
3074 0 : v = ((char*)ptr)[ii];
3075 0 : break;
3076 0 : case TID_WORD:
3077 0 : v = ((unsigned short *)ptr)[ii];
3078 0 : break;
3079 0 : case TID_SHORT:
3080 0 : v = ((signed short *)ptr)[ii];
3081 0 : break;
3082 0 : case TID_DWORD:
3083 0 : v = ((unsigned int *)ptr)[ii];
3084 0 : break;
3085 0 : case TID_INT:
3086 0 : v = ((int *)ptr)[ii];
3087 0 : break;
3088 0 : case TID_BOOL:
3089 0 : v = ((unsigned int *)ptr)[ii];
3090 0 : break;
3091 0 : case TID_FLOAT:
3092 0 : v = ((float*)ptr)[ii];
3093 0 : break;
3094 0 : case TID_DOUBLE:
3095 0 : v = ((double*)ptr)[ii];
3096 0 : break;
3097 : }
3098 :
3099 0 : buffer[i]->Add(t, v);
3100 0 : last_time[i] = t;
3101 : }
3102 0 : count++;
3103 0 : }
3104 :
3105 0 : delete[] buf;
3106 :
3107 0 : ::close(fd);
3108 :
3109 0 : if (debug) {
3110 0 : printf("FileHistory::read_data: file %s map", s->fFileName.c_str());
3111 0 : for (size_t i=0; i<var_schema_index.size(); i++) {
3112 0 : printf(" %2d", var_schema_index[i]);
3113 : }
3114 0 : printf(" read %d rows\n", count);
3115 : }
3116 :
3117 0 : if (debug)
3118 0 : printf("FileHistory::read: file %s, schema time %s..%s, read time %s..%s, %d vars, read %d rows\n", s->fFileName.c_str(), TimeToString(s->fTimeFrom).c_str(), TimeToString(s->fTimeTo).c_str(), TimeToString(start_time).c_str(), TimeToString(end_time).c_str(), num_var, count);
3119 :
3120 0 : return HS_SUCCESS;
3121 : }
3122 :
3123 : ////////////////////////////////////////////////////////
3124 : // Implementation of the MidasHistoryInterface //
3125 : ////////////////////////////////////////////////////////
3126 :
3127 : class SchemaHistoryBase: public MidasHistoryInterface
3128 : {
3129 : protected:
3130 : int fDebug = 0;
3131 : std::string fConnectString;
3132 :
3133 : // writer data
3134 : HsSchemaVector fWriterSchema;
3135 : std::vector<HsSchema*> fWriterEvents;
3136 :
3137 : // reader data
3138 : HsSchemaVector fReaderSchema;
3139 :
3140 : public:
3141 0 : SchemaHistoryBase()
3142 0 : {
3143 : // empty
3144 0 : }
3145 :
3146 0 : virtual ~SchemaHistoryBase()
3147 0 : {
3148 0 : for (size_t i=0; i<fWriterEvents.size(); i++) {
3149 : //printf("fWriterEvents[%zu] is %p\n", i, fWriterEvents[i]);
3150 0 : if (fWriterEvents[i]) {
3151 0 : delete fWriterEvents[i];
3152 0 : fWriterEvents[i] = NULL;
3153 : }
3154 : }
3155 0 : fWriterEvents.clear();
3156 0 : }
3157 :
3158 0 : virtual int hs_set_debug(int debug)
3159 : {
3160 0 : int old = fDebug;
3161 0 : fDebug = debug;
3162 0 : return old;
3163 : }
3164 :
3165 : virtual int hs_connect(const char* connect_string) = 0;
3166 : virtual int hs_disconnect() = 0;
3167 :
3168 : protected:
3169 : ////////////////////////////////////////////////////////
3170 : // Schema functions //
3171 : ////////////////////////////////////////////////////////
3172 :
3173 : // load existing schema
3174 : virtual int read_schema(HsSchemaVector* sv, const char* event_name, const time_t timestamp) = 0; // event_name: =NULL means read only event names, =event_name means load tag names for all matching events; timestamp: =0 means read all schema all the way to the beginning of time, =time means read schema in effect at this time and all newer schema
3175 :
3176 : // return a new copy of a schema for writing into history. If schema for this event does not exist, it will be created
3177 : virtual HsSchema* new_event(const char* event_name, time_t timestamp, int ntags, const TAG tags[]) = 0;
3178 :
3179 : // if output file is too big, close it, create a new output file
3180 : virtual HsSchema* maybe_reopen(const char* event_name, time_t timestamp, HsSchema* s) = 0;
3181 :
3182 : public:
3183 : ////////////////////////////////////////////////////////
3184 : // Functions used by mlogger //
3185 : ////////////////////////////////////////////////////////
3186 :
3187 : int hs_define_event(const char* event_name, time_t timestamp, int ntags, const TAG tags[]);
3188 : int hs_write_event(const char* event_name, time_t timestamp, int buffer_size, const char* buffer);
3189 : int hs_flush_buffers();
3190 :
3191 : ////////////////////////////////////////////////////////
3192 : // Functions used by mhttpd //
3193 : ////////////////////////////////////////////////////////
3194 :
3195 : int hs_clear_cache();
3196 : int hs_get_events(time_t t, std::vector<std::string> *pevents);
3197 : int hs_get_tags(const char* event_name, time_t t, std::vector<TAG> *ptags);
3198 : int hs_get_last_written(time_t timestamp, int num_var, const char* const event_name[], const char* const var_name[], const int var_index[], time_t last_written[]);
3199 : int hs_read_buffer(time_t start_time, time_t end_time,
3200 : int num_var, const char* const event_name[], const char* const var_name[], const int var_index[],
3201 : MidasHistoryBufferInterface* buffer[],
3202 : int hs_status[]);
3203 :
3204 : /*------------------------------------------------------------------*/
3205 :
3206 : class ReadBuffer: public MidasHistoryBufferInterface
3207 : {
3208 : public:
3209 : time_t fFirstTime;
3210 : time_t fLastTime;
3211 : time_t fInterval;
3212 :
3213 : int fNumAdded;
3214 :
3215 : int fNumAlloc;
3216 : int *fNumEntries;
3217 : time_t **fTimeBuffer;
3218 : double **fDataBuffer;
3219 :
3220 : time_t fPrevTime;
3221 :
3222 0 : ReadBuffer(time_t first_time, time_t last_time, time_t interval) // ctor
3223 0 : {
3224 0 : fNumAdded = 0;
3225 :
3226 0 : fFirstTime = first_time;
3227 0 : fLastTime = last_time;
3228 0 : fInterval = interval;
3229 :
3230 0 : fNumAlloc = 0;
3231 0 : fNumEntries = NULL;
3232 0 : fTimeBuffer = NULL;
3233 0 : fDataBuffer = NULL;
3234 :
3235 0 : fPrevTime = 0;
3236 0 : }
3237 :
3238 0 : ~ReadBuffer() // dtor
3239 0 : {
3240 0 : }
3241 :
3242 0 : void Realloc(int wantalloc)
3243 : {
3244 0 : if (wantalloc < fNumAlloc - 10)
3245 0 : return;
3246 :
3247 0 : int newalloc = fNumAlloc*2;
3248 :
3249 0 : if (newalloc <= 1000)
3250 0 : newalloc = wantalloc + 1000;
3251 :
3252 : //printf("wantalloc %d, fNumEntries %d, fNumAlloc %d, newalloc %d\n", wantalloc, *fNumEntries, fNumAlloc, newalloc);
3253 :
3254 0 : *fTimeBuffer = (time_t*)realloc(*fTimeBuffer, sizeof(time_t)*newalloc);
3255 0 : assert(*fTimeBuffer);
3256 :
3257 0 : *fDataBuffer = (double*)realloc(*fDataBuffer, sizeof(double)*newalloc);
3258 0 : assert(*fDataBuffer);
3259 :
3260 0 : fNumAlloc = newalloc;
3261 : }
3262 :
3263 0 : void Add(time_t t, double v)
3264 : {
3265 0 : if (t < fFirstTime)
3266 0 : return;
3267 0 : if (t > fLastTime)
3268 0 : return;
3269 :
3270 0 : fNumAdded++;
3271 :
3272 0 : if ((fPrevTime==0) || (t >= fPrevTime + fInterval)) {
3273 0 : int pos = *fNumEntries;
3274 :
3275 0 : Realloc(pos + 1);
3276 :
3277 0 : (*fTimeBuffer)[pos] = t;
3278 0 : (*fDataBuffer)[pos] = v;
3279 :
3280 0 : (*fNumEntries) = pos + 1;
3281 :
3282 0 : fPrevTime = t;
3283 : }
3284 : }
3285 :
3286 0 : void Finish()
3287 : {
3288 :
3289 0 : }
3290 : };
3291 :
3292 : /*------------------------------------------------------------------*/
3293 :
3294 : int hs_read(time_t start_time, time_t end_time, time_t interval,
3295 : int num_var,
3296 : const char* const event_name[], const char* const var_name[], const int var_index[],
3297 : int num_entries[],
3298 : time_t* time_buffer[], double* data_buffer[],
3299 : int st[]);
3300 : /*------------------------------------------------------------------*/
3301 :
3302 :
3303 : int hs_read_binned(time_t start_time, time_t end_time, int num_bins,
3304 : int num_var, const char* const event_name[], const char* const var_name[], const int var_index[],
3305 : int num_entries[],
3306 : int* count_bins[], double* mean_bins[], double* rms_bins[], double* min_bins[], double* max_bins[],
3307 : time_t* bins_first_time[], double* bins_first_value[],
3308 : time_t* bins_last_time[], double* bins_last_value[],
3309 : time_t last_time[], double last_value[],
3310 : int st[]);
3311 : };
3312 :
3313 0 : MidasHistoryBinnedBuffer::MidasHistoryBinnedBuffer(time_t first_time, time_t last_time, int num_bins) // ctor
3314 : {
3315 0 : fNumEntries = 0;
3316 :
3317 0 : fNumBins = num_bins;
3318 0 : fFirstTime = first_time;
3319 0 : fLastTime = last_time;
3320 :
3321 0 : fSum0 = new double[num_bins];
3322 0 : fSum1 = new double[num_bins];
3323 0 : fSum2 = new double[num_bins];
3324 :
3325 0 : for (int i=0; i<num_bins; i++) {
3326 0 : fSum0[i] = 0;
3327 0 : fSum1[i] = 0;
3328 0 : fSum2[i] = 0;
3329 : }
3330 0 : }
3331 :
3332 0 : MidasHistoryBinnedBuffer::~MidasHistoryBinnedBuffer() // dtor
3333 : {
3334 0 : delete fSum0; fSum0 = NULL;
3335 0 : delete fSum1; fSum1 = NULL;
3336 0 : delete fSum2; fSum2 = NULL;
3337 : // poison the pointers
3338 0 : fCount = NULL;
3339 0 : fMean = NULL;
3340 0 : fRms = NULL;
3341 0 : fMin = NULL;
3342 0 : fMax = NULL;
3343 0 : fBinsFirstTime = NULL;
3344 0 : fBinsFirstValue = NULL;
3345 0 : fBinsLastTime = NULL;
3346 0 : fBinsLastValue = NULL;
3347 0 : fLastTimePtr = NULL;
3348 0 : fLastValuePtr = NULL;
3349 0 : }
3350 :
3351 0 : void MidasHistoryBinnedBuffer::Start()
3352 : {
3353 0 : for (int ibin = 0; ibin < fNumBins; ibin++) {
3354 0 : if (fMin)
3355 0 : fMin[ibin] = 0;
3356 0 : if (fMax)
3357 0 : fMax[ibin] = 0;
3358 0 : if (fBinsFirstTime)
3359 0 : fBinsFirstTime[ibin] = 0;
3360 0 : if (fBinsFirstValue)
3361 0 : fBinsFirstValue[ibin] = 0;
3362 0 : if (fBinsLastTime)
3363 0 : fBinsLastTime[ibin] = 0;
3364 0 : if (fBinsLastValue)
3365 0 : fBinsLastValue[ibin] = 0;
3366 : }
3367 0 : if (fLastTimePtr)
3368 0 : *fLastTimePtr = 0;
3369 0 : if (fLastValuePtr)
3370 0 : *fLastValuePtr = 0;
3371 0 : }
3372 :
3373 0 : void MidasHistoryBinnedBuffer::Add(time_t t, double v)
3374 : {
3375 0 : if (t < fFirstTime)
3376 0 : return;
3377 0 : if (t > fLastTime)
3378 0 : return;
3379 :
3380 0 : fNumEntries++;
3381 :
3382 0 : double a = (double)(t - fFirstTime);
3383 0 : double b = (double)(fLastTime - fFirstTime);
3384 0 : double fbin = fNumBins*a/b;
3385 :
3386 0 : int ibin = (int)fbin;
3387 :
3388 0 : if (ibin < 0)
3389 0 : ibin = 0;
3390 0 : else if (ibin >= fNumBins)
3391 0 : ibin = fNumBins-1;
3392 :
3393 0 : if (fSum0[ibin] == 0) {
3394 0 : if (fMin)
3395 0 : fMin[ibin] = v;
3396 0 : if (fMax)
3397 0 : fMax[ibin] = v;
3398 0 : if (fBinsFirstTime)
3399 0 : fBinsFirstTime[ibin] = t;
3400 0 : if (fBinsFirstValue)
3401 0 : fBinsFirstValue[ibin] = v;
3402 0 : if (fBinsLastTime)
3403 0 : fBinsLastTime[ibin] = t;
3404 0 : if (fBinsLastValue)
3405 0 : fBinsLastValue[ibin] = v;
3406 0 : if (fLastTimePtr)
3407 0 : *fLastTimePtr = t;
3408 0 : if (fLastValuePtr)
3409 0 : *fLastValuePtr = v;
3410 : }
3411 :
3412 0 : fSum0[ibin] += 1.0;
3413 0 : fSum1[ibin] += v;
3414 0 : fSum2[ibin] += v*v;
3415 :
3416 0 : if (fMin)
3417 0 : if (v < fMin[ibin])
3418 0 : fMin[ibin] = v;
3419 :
3420 0 : if (fMax)
3421 0 : if (v > fMax[ibin])
3422 0 : fMax[ibin] = v;
3423 :
3424 : // NOTE: this assumes t and v are sorted by time.
3425 0 : if (fBinsLastTime)
3426 0 : fBinsLastTime[ibin] = t;
3427 0 : if (fBinsLastValue)
3428 0 : fBinsLastValue[ibin] = v;
3429 :
3430 0 : if (fLastTimePtr)
3431 0 : if (t > *fLastTimePtr) {
3432 0 : *fLastTimePtr = t;
3433 0 : if (fLastValuePtr)
3434 0 : *fLastValuePtr = v;
3435 : }
3436 : }
3437 :
3438 0 : void MidasHistoryBinnedBuffer::Finish()
3439 : {
3440 0 : for (int i=0; i<fNumBins; i++) {
3441 0 : double num = fSum0[i];
3442 0 : double mean = 0;
3443 0 : double variance = 0;
3444 0 : if (num > 0) {
3445 0 : mean = fSum1[i]/num;
3446 0 : variance = fSum2[i]/num-mean*mean;
3447 : }
3448 0 : double rms = 0;
3449 0 : if (variance > 0)
3450 0 : rms = sqrt(variance);
3451 :
3452 0 : if (fCount)
3453 0 : fCount[i] = (int)num;
3454 :
3455 0 : if (fMean)
3456 0 : fMean[i] = mean;
3457 :
3458 0 : if (fRms)
3459 0 : fRms[i] = rms;
3460 :
3461 0 : if (num == 0) {
3462 0 : if (fMin)
3463 0 : fMin[i] = 0;
3464 0 : if (fMax)
3465 0 : fMax[i] = 0;
3466 : }
3467 : }
3468 0 : }
3469 :
3470 0 : int SchemaHistoryBase::hs_define_event(const char* event_name, time_t timestamp, int ntags, const TAG tags[])
3471 : {
3472 0 : if (fDebug) {
3473 0 : printf("hs_define_event: event name [%s] with %d tags\n", event_name, ntags);
3474 0 : if (fDebug > 1)
3475 0 : PrintTags(ntags, tags);
3476 : }
3477 :
3478 : // delete all events with the same name
3479 0 : for (size_t i=0; i<fWriterEvents.size(); i++)
3480 0 : if (fWriterEvents[i])
3481 0 : if (event_name_cmp(fWriterEvents[i]->fEventName, event_name)==0) {
3482 0 : if (fDebug)
3483 0 : printf("deleting exising event %s\n", event_name);
3484 0 : fWriterEvents[i]->close();
3485 0 : delete fWriterEvents[i];
3486 0 : fWriterEvents[i] = NULL;
3487 : }
3488 :
3489 : // check for wrong types etc
3490 0 : for (int i=0; i<ntags; i++) {
3491 0 : if (strlen(tags[i].name) < 1) {
3492 0 : cm_msg(MERROR, "hs_define_event", "Error: History event \'%s\' has empty name at index %d", event_name, i);
3493 0 : return HS_FILE_ERROR;
3494 : }
3495 0 : if (tags[i].name[0] == ' ') {
3496 0 : cm_msg(MERROR, "hs_define_event", "Error: History event \'%s\' has name \'%s\' starting with a blank", event_name, tags[i].name);
3497 0 : return HS_FILE_ERROR;
3498 : }
3499 0 : if (tags[i].type <= 0 || tags[i].type >= TID_LAST) {
3500 0 : cm_msg(MERROR, "hs_define_event", "Error: History event \'%s\' tag \'%s\' at index %d has invalid type %d",
3501 0 : event_name, tags[i].name, i, tags[i].type);
3502 0 : return HS_FILE_ERROR;
3503 : }
3504 0 : if (tags[i].type == TID_STRING) {
3505 0 : cm_msg(MERROR, "hs_define_event",
3506 : "Error: History event \'%s\' tag \'%s\' at index %d has forbidden type TID_STRING", event_name,
3507 0 : tags[i].name, i);
3508 0 : return HS_FILE_ERROR;
3509 : }
3510 0 : if (tags[i].n_data <= 0) {
3511 0 : cm_msg(MERROR, "hs_define_event", "Error: History event \'%s\' tag \'%s\' at index %d has invalid n_data %d",
3512 0 : event_name, tags[i].name, i, tags[i].n_data);
3513 0 : return HS_FILE_ERROR;
3514 : }
3515 : }
3516 :
3517 : // check for duplicate names. Done by sorting, since this takes only O(N*log*N))
3518 0 : std::vector<std::string> names;
3519 0 : for (int i=0; i<ntags; i++) {
3520 0 : std::string str(tags[i].name);
3521 0 : std::transform(str.begin(), str.end(), str.begin(), ::toupper);
3522 0 : names.push_back(str);
3523 0 : }
3524 0 : std::sort(names.begin(), names.end());
3525 0 : for (int i=0; i<ntags-1; i++) {
3526 0 : if (names[i] == names[i + 1]) {
3527 0 : cm_msg(MERROR, "hs_define_event",
3528 : "Error: History event \'%s\' has duplicate tag name \'%s\'", event_name,
3529 0 : names[i].c_str());
3530 0 : return HS_FILE_ERROR;
3531 : }
3532 : }
3533 :
3534 0 : HsSchema* s = new_event(event_name, timestamp, ntags, tags);
3535 0 : if (!s)
3536 0 : return HS_FILE_ERROR;
3537 :
3538 0 : s->fDisabled = false;
3539 :
3540 0 : s->remove_inactive_columns();
3541 :
3542 : // find empty slot in events list
3543 0 : for (size_t i=0; i<fWriterEvents.size(); i++)
3544 0 : if (!fWriterEvents[i]) {
3545 0 : fWriterEvents[i] = s;
3546 0 : s = NULL;
3547 0 : break;
3548 : }
3549 :
3550 : // if no empty slots, add at the end
3551 0 : if (s)
3552 0 : fWriterEvents.push_back(s);
3553 :
3554 0 : return HS_SUCCESS;
3555 0 : }
3556 :
3557 0 : int SchemaHistoryBase::hs_write_event(const char* event_name, time_t timestamp, int xbuffer_size, const char* buffer)
3558 : {
3559 0 : if (fDebug)
3560 0 : printf("hs_write_event: write event \'%s\', time %d, size %d\n", event_name, (int)timestamp, xbuffer_size);
3561 :
3562 0 : assert(xbuffer_size > 0);
3563 :
3564 0 : size_t buffer_size = xbuffer_size;
3565 :
3566 0 : HsSchema *s = NULL;
3567 :
3568 : // find this event
3569 0 : for (size_t i=0; i<fWriterEvents.size(); i++)
3570 0 : if (fWriterEvents[i] && (event_name_cmp(fWriterEvents[i]->fEventName, event_name)==0)) {
3571 0 : s = fWriterEvents[i];
3572 0 : break;
3573 : }
3574 :
3575 : // not found
3576 0 : if (!s)
3577 0 : return HS_UNDEFINED_EVENT;
3578 :
3579 : // deactivated because of error?
3580 0 : if (s->fDisabled) {
3581 : //printf("HERE!\n");
3582 0 : return HS_FILE_ERROR;
3583 : }
3584 :
3585 0 : s = maybe_reopen(event_name, timestamp, s);
3586 :
3587 0 : assert(s != NULL);
3588 :
3589 0 : if (s->fNumBytes == 0) { // compute expected data size
3590 : // NB: history data does not have any padding!
3591 0 : for (size_t i=0; i<s->fVariables.size(); i++) {
3592 0 : s->fNumBytes += s->fVariables[i].n_bytes;
3593 : }
3594 : }
3595 :
3596 : int status;
3597 :
3598 0 : if (buffer_size > s->fNumBytes) { // too many bytes!
3599 0 : if (s->fCountWriteOversize == 0) {
3600 : // only report first occurence
3601 : // count of all occurences is reported by HsSchema destructor
3602 0 : cm_msg(MERROR, "hs_write_event", "Event \'%s\' data size mismatch: expected %zu bytes, got %zu bytes", s->fEventName.c_str(), s->fNumBytes, buffer_size);
3603 : }
3604 0 : s->fCountWriteOversize++;
3605 0 : if (buffer_size > s->fWriteMaxSize)
3606 0 : s->fWriteMaxSize = buffer_size;
3607 0 : status = s->write_event(timestamp, buffer, s->fNumBytes);
3608 0 : } else if (buffer_size < s->fNumBytes) { // too few bytes
3609 0 : if (s->fCountWriteUndersize == 0) {
3610 : // only report first occurence
3611 : // count of all occurences is reported by HsSchema destructor
3612 0 : cm_msg(MERROR, "hs_write_event", "Event \'%s\' data size mismatch: expected %zu bytes, got %zu bytes", s->fEventName.c_str(), s->fNumBytes, buffer_size);
3613 : }
3614 0 : s->fCountWriteUndersize++;
3615 0 : if (s->fWriteMinSize == 0)
3616 0 : s->fWriteMinSize = buffer_size;
3617 0 : else if (buffer_size < s->fWriteMinSize)
3618 0 : s->fWriteMinSize = buffer_size;
3619 0 : char* tmp = (char*)malloc(s->fNumBytes);
3620 0 : memcpy(tmp, buffer, buffer_size);
3621 0 : memset(tmp + buffer_size, 0, s->fNumBytes - buffer_size);
3622 0 : status = s->write_event(timestamp, tmp, s->fNumBytes);
3623 0 : free(tmp);
3624 : } else {
3625 0 : assert(buffer_size == s->fNumBytes); // obviously
3626 0 : status = s->write_event(timestamp, buffer, buffer_size);
3627 : }
3628 :
3629 : // if could not write event, deactivate it
3630 0 : if (status != HS_SUCCESS) {
3631 0 : s->fDisabled = true;
3632 0 : cm_msg(MERROR, "hs_write_event", "Event \'%s\' disabled after write error %d", event_name, status);
3633 0 : return HS_FILE_ERROR;
3634 : }
3635 :
3636 0 : return HS_SUCCESS;
3637 : }
3638 :
3639 0 : int SchemaHistoryBase::hs_flush_buffers()
3640 : {
3641 0 : int status = HS_SUCCESS;
3642 :
3643 0 : if (fDebug)
3644 0 : printf("hs_flush_buffers!\n");
3645 :
3646 0 : for (size_t i=0; i<fWriterEvents.size(); i++)
3647 0 : if (fWriterEvents[i]) {
3648 0 : int xstatus = fWriterEvents[i]->flush_buffers();
3649 0 : if (xstatus != HS_SUCCESS)
3650 0 : status = xstatus;
3651 : }
3652 :
3653 0 : return status;
3654 : }
3655 :
3656 : ////////////////////////////////////////////////////////
3657 : // Functions used by mhttpd //
3658 : ////////////////////////////////////////////////////////
3659 :
3660 0 : int SchemaHistoryBase::hs_clear_cache()
3661 : {
3662 0 : if (fDebug)
3663 0 : printf("SchemaHistoryBase::hs_clear_cache!\n");
3664 :
3665 0 : fWriterSchema.clear();
3666 0 : fReaderSchema.clear();
3667 :
3668 0 : return HS_SUCCESS;
3669 : }
3670 :
3671 0 : int SchemaHistoryBase::hs_get_events(time_t t, std::vector<std::string> *pevents)
3672 : {
3673 0 : if (fDebug)
3674 0 : printf("hs_get_events, time %s\n", TimeToString(t).c_str());
3675 :
3676 0 : int status = read_schema(&fReaderSchema, NULL, t);
3677 0 : if (status != HS_SUCCESS)
3678 0 : return status;
3679 :
3680 0 : if (fDebug) {
3681 0 : printf("hs_get_events: available schema:\n");
3682 0 : fReaderSchema.print(false);
3683 : }
3684 :
3685 0 : assert(pevents);
3686 :
3687 0 : for (size_t i=0; i<fReaderSchema.size(); i++) {
3688 0 : HsSchema* s = fReaderSchema[i];
3689 0 : if (t && s->fTimeTo && s->fTimeTo < t)
3690 0 : continue;
3691 0 : bool dupe = false;
3692 0 : for (size_t j=0; j<pevents->size(); j++)
3693 0 : if (event_name_cmp((*pevents)[j], s->fEventName.c_str())==0) {
3694 0 : dupe = true;
3695 0 : break;
3696 : }
3697 :
3698 0 : if (!dupe)
3699 0 : pevents->push_back(s->fEventName);
3700 : }
3701 :
3702 0 : std::sort(pevents->begin(), pevents->end());
3703 :
3704 0 : if (fDebug) {
3705 0 : printf("hs_get_events: returning %zu events\n", pevents->size());
3706 0 : for (size_t i=0; i<pevents->size(); i++) {
3707 0 : printf(" %zu: [%s]\n", i, (*pevents)[i].c_str());
3708 : }
3709 : }
3710 :
3711 0 : return HS_SUCCESS;
3712 : }
3713 :
3714 0 : int SchemaHistoryBase::hs_get_tags(const char* event_name, time_t t, std::vector<TAG> *ptags)
3715 : {
3716 0 : if (fDebug)
3717 0 : printf("hs_get_tags: event [%s], time %s\n", event_name, TimeToString(t).c_str());
3718 :
3719 0 : assert(ptags);
3720 :
3721 0 : int status = read_schema(&fReaderSchema, event_name, t);
3722 0 : if (status != HS_SUCCESS)
3723 0 : return status;
3724 :
3725 0 : bool found_event = false;
3726 0 : for (size_t i=0; i<fReaderSchema.size(); i++) {
3727 0 : HsSchema* s = fReaderSchema[i];
3728 0 : if (t && s->fTimeTo && s->fTimeTo < t)
3729 0 : continue;
3730 :
3731 0 : if (event_name_cmp(s->fEventName, event_name) != 0)
3732 0 : continue;
3733 :
3734 0 : found_event = true;
3735 :
3736 0 : for (size_t i=0; i<s->fVariables.size(); i++) {
3737 0 : const char* tagname = s->fVariables[i].name.c_str();
3738 : //printf("event_name [%s], table_name [%s], column name [%s], tag name [%s]\n", event_name, tn.c_str(), cn.c_str(), tagname);
3739 :
3740 0 : bool dupe = false;
3741 :
3742 0 : for (size_t k=0; k<ptags->size(); k++)
3743 0 : if (strcasecmp((*ptags)[k].name, tagname) == 0) {
3744 0 : dupe = true;
3745 0 : break;
3746 : }
3747 :
3748 0 : if (!dupe) {
3749 : TAG t;
3750 0 : mstrlcpy(t.name, tagname, sizeof(t.name));
3751 0 : t.type = s->fVariables[i].type;
3752 0 : t.n_data = s->fVariables[i].n_data;
3753 :
3754 0 : ptags->push_back(t);
3755 : }
3756 : }
3757 : }
3758 :
3759 0 : if (!found_event)
3760 0 : return HS_UNDEFINED_EVENT;
3761 :
3762 0 : if (fDebug) {
3763 0 : printf("hs_get_tags: event [%s], returning %zu tags\n", event_name, ptags->size());
3764 0 : for (size_t i=0; i<ptags->size(); i++) {
3765 0 : printf(" tag[%zu]: %s[%d] type %d\n", i, (*ptags)[i].name, (*ptags)[i].n_data, (*ptags)[i].type);
3766 : }
3767 : }
3768 :
3769 0 : return HS_SUCCESS;
3770 : }
3771 :
3772 0 : int SchemaHistoryBase::hs_get_last_written(time_t timestamp, int num_var, const char* const event_name[], const char* const var_name[], const int var_index[], time_t last_written[])
3773 : {
3774 0 : if (fDebug) {
3775 0 : printf("hs_get_last_written: timestamp %s, num_var %d\n", TimeToString(timestamp).c_str(), num_var);
3776 : }
3777 :
3778 0 : for (int j=0; j<num_var; j++) {
3779 0 : last_written[j] = 0;
3780 : }
3781 :
3782 0 : for (int i=0; i<num_var; i++) {
3783 0 : int status = read_schema(&fReaderSchema, event_name[i], 0);
3784 0 : if (status != HS_SUCCESS)
3785 0 : return status;
3786 : }
3787 :
3788 : //fReaderSchema.print(false);
3789 :
3790 0 : for (int i=0; i<num_var; i++) {
3791 0 : for (size_t ss=0; ss<fReaderSchema.size(); ss++) {
3792 0 : HsSchema* s = fReaderSchema[ss];
3793 : // schema is too new
3794 0 : if (s->fTimeFrom && s->fTimeFrom >= timestamp)
3795 0 : continue;
3796 : // this schema is newer than last_written and may contain newer data?
3797 0 : if (s->fTimeFrom && s->fTimeFrom < last_written[i])
3798 0 : continue;
3799 : // schema for the variables we want?
3800 0 : int sindex = s->match_event_var(event_name[i], var_name[i], var_index[i]);
3801 0 : if (sindex < 0)
3802 0 : continue;
3803 :
3804 0 : time_t lw = 0;
3805 :
3806 0 : int status = s->read_last_written(timestamp, fDebug, &lw);
3807 :
3808 0 : if (status == HS_SUCCESS && lw != 0) {
3809 0 : for (int j=0; j<num_var; j++) {
3810 0 : int sj = s->match_event_var(event_name[j], var_name[j], var_index[j]);
3811 0 : if (sj < 0)
3812 0 : continue;
3813 :
3814 0 : if (lw > last_written[j])
3815 0 : last_written[j] = lw;
3816 : }
3817 : }
3818 : }
3819 : }
3820 :
3821 0 : if (fDebug) {
3822 0 : printf("hs_get_last_written: timestamp time %s, num_var %d, result:\n", TimeToString(timestamp).c_str(), num_var);
3823 0 : for (int i=0; i<num_var; i++) {
3824 0 : printf(" event [%s] tag [%s] index [%d] last_written %s\n", event_name[i], var_name[i], var_index[i], TimeToString(last_written[i]).c_str());
3825 : }
3826 : }
3827 :
3828 0 : return HS_SUCCESS;
3829 : }
3830 :
3831 0 : int SchemaHistoryBase::hs_read_buffer(time_t start_time, time_t end_time,
3832 : int num_var, const char* const event_name[], const char* const var_name[], const int var_index[],
3833 : MidasHistoryBufferInterface* buffer[],
3834 : int hs_status[])
3835 : {
3836 0 : if (fDebug)
3837 0 : printf("hs_read_buffer: %d variables, start time %s, end time %s\n", num_var, TimeToString(start_time).c_str(), TimeToString(end_time).c_str());
3838 :
3839 0 : for (int i=0; i<num_var; i++) {
3840 0 : int status = read_schema(&fReaderSchema, event_name[i], start_time);
3841 0 : if (status != HS_SUCCESS)
3842 0 : return status;
3843 : }
3844 :
3845 : #if 0
3846 : if (fDebug)
3847 : fReaderSchema.print(false);
3848 : #endif
3849 :
3850 0 : for (int i=0; i<num_var; i++) {
3851 0 : hs_status[i] = HS_UNDEFINED_VAR;
3852 : }
3853 :
3854 : //for (size_t ss=0; ss<fReaderSchema.size(); ss++) {
3855 : // HsSchema* s = fReaderSchema[ss];
3856 : // HsFileSchema* fs = dynamic_cast<HsFileSchema*>(s);
3857 : // assert(fs != NULL);
3858 : // printf("schema %d from %s to %s, filename %s\n", ss, TimeToString(fs->fTimeFrom).c_str(), TimeToString(fs->fTimeTo).c_str(), fs->fFileName.c_str());
3859 : //}
3860 :
3861 : // check that schema are sorted by time
3862 :
3863 : #if 0
3864 : // check that schema list is sorted by time, descending fTimeFrom, newest schema first
3865 : for (size_t ss=0; ss<fReaderSchema.size(); ss++) {
3866 : if (fDebug) {
3867 : //printf("Check schema %zu/%zu: prev from %s, this from %s to %s, compare %d %d %d\n", ss, fReaderSchema.size(),
3868 : // TimeToString(fReaderSchema[ss-1]->fTimeFrom).c_str(),
3869 : // TimeToString(fReaderSchema[ss]->fTimeFrom).c_str(),
3870 : // TimeToString(fReaderSchema[ss]->fTimeTo).c_str(),
3871 : // fReaderSchema[ss-1]->fTimeFrom >= fReaderSchema[ss]->fTimeTo,
3872 : // fReaderSchema[ss-1]->fTimeFrom > fReaderSchema[ss]->fTimeFrom,
3873 : // (fReaderSchema[ss-1]->fTimeFrom >= fReaderSchema[ss]->fTimeTo) && (fReaderSchema[ss-1]->fTimeFrom > fReaderSchema[ss]->fTimeFrom));
3874 : printf("Schema %zu/%zu: from %s to %s, name %s\n", ss, fReaderSchema.size(),
3875 : TimeToString(fReaderSchema[ss]->fTimeFrom).c_str(),
3876 : TimeToString(fReaderSchema[ss]->fTimeTo).c_str(),
3877 : fReaderSchema[ss]->fEventName.c_str());
3878 : }
3879 :
3880 : if (ss > 0) {
3881 : //if ((fReaderSchema[ss-1]->fTimeFrom >= fReaderSchema[ss]->fTimeTo) && (fReaderSchema[ss-1]->fTimeFrom > fReaderSchema[ss]->fTimeFrom)) {
3882 : if ((fReaderSchema[ss-1]->fTimeFrom >= fReaderSchema[ss]->fTimeFrom)) {
3883 : // good
3884 : } else {
3885 : cm_msg(MERROR, "SchemaHistoryBase::hs_read_buffer", "History internal error, schema is not ordered by time. Please report this error to the midas forum.");
3886 : return HS_FILE_ERROR;
3887 : }
3888 : }
3889 : }
3890 : #endif
3891 :
3892 0 : std::vector<HsSchema*> slist;
3893 0 : std::vector<std::vector<int>> smap;
3894 :
3895 0 : for (size_t ss=0; ss<fReaderSchema.size(); ss++) {
3896 0 : HsSchema* s = fReaderSchema[ss];
3897 : // schema is too new?
3898 0 : if (s->fTimeFrom && s->fTimeFrom > end_time)
3899 0 : continue;
3900 : // schema is too old
3901 0 : if (s->fTimeTo && s->fTimeTo < start_time)
3902 0 : continue;
3903 :
3904 0 : std::vector<int> sm;
3905 :
3906 0 : for (int i=0; i<num_var; i++) {
3907 : // schema for the variables we want?
3908 0 : int sindex = s->match_event_var(event_name[i], var_name[i], var_index[i]);
3909 0 : if (sindex < 0)
3910 0 : continue;
3911 :
3912 0 : if (sm.empty()) {
3913 0 : for (int i=0; i<num_var; i++) {
3914 0 : sm.push_back(-1);
3915 : }
3916 : }
3917 :
3918 0 : sm[i] = sindex;
3919 : }
3920 :
3921 0 : if (!sm.empty()) {
3922 0 : slist.push_back(s);
3923 0 : smap.push_back(sm);
3924 : }
3925 0 : }
3926 :
3927 0 : if (0||fDebug) {
3928 0 : printf("Found %zu matching schema:\n", slist.size());
3929 :
3930 0 : for (size_t i=0; i<slist.size(); i++) {
3931 0 : HsSchema* s = slist[i];
3932 0 : s->print();
3933 0 : for (int k=0; k<num_var; k++)
3934 0 : printf(" tag %s[%d] sindex %d\n", var_name[k], var_index[k], smap[i][k]);
3935 : }
3936 : }
3937 :
3938 : //for (size_t ss=0; ss<slist.size(); ss++) {
3939 : // HsSchema* s = slist[ss];
3940 : // HsFileSchema* fs = dynamic_cast<HsFileSchema*>(s);
3941 : // assert(fs != NULL);
3942 : // printf("schema %zu from %s to %s, filename %s", ss, TimeToString(fs->fTimeFrom).c_str(), TimeToString(fs->fTimeTo).c_str(), fs->fFileName.c_str());
3943 : // printf(" smap ");
3944 : // for (int k=0; k<num_var; k++)
3945 : // printf(" %2d", smap[ss][k]);
3946 : // printf("\n");
3947 : //}
3948 :
3949 0 : for (size_t ss=1; ss<slist.size(); ss++) {
3950 0 : if (fDebug) {
3951 0 : printf("Check schema %zu/%zu: prev from %s, this from %s to %s, compare %d\n", ss, slist.size(),
3952 0 : TimeToString(slist[ss-1]->fTimeFrom).c_str(),
3953 0 : TimeToString(slist[ss]->fTimeFrom).c_str(),
3954 0 : TimeToString(slist[ss]->fTimeTo).c_str(),
3955 0 : slist[ss-1]->fTimeFrom >= slist[ss]->fTimeFrom);
3956 : }
3957 0 : if (slist[ss-1]->fTimeFrom >= slist[ss]->fTimeFrom) {
3958 : // good
3959 : } else {
3960 0 : cm_msg(MERROR, "SchemaHistoryBase::hs_read_buffer", "History internal error, selected schema is not ordered by time. Please report this error to the midas forum.");
3961 0 : return HS_FILE_ERROR;
3962 : }
3963 : }
3964 :
3965 0 : std::vector<time_t> last_time;
3966 :
3967 0 : for (int i=0; i<num_var; i++) {
3968 0 : last_time.push_back(start_time);
3969 : }
3970 :
3971 0 : if (slist.size() > 0) {
3972 0 : for (size_t i=slist.size()-1; ; i--) {
3973 0 : HsSchema* s = slist[i];
3974 :
3975 0 : int status = s->read_data(start_time, end_time, num_var, smap[i], var_index, fDebug, last_time, buffer);
3976 :
3977 0 : if (status == HS_SUCCESS) {
3978 0 : for (int j=0; j<num_var; j++) {
3979 0 : if (smap[i][j] >= 0)
3980 0 : hs_status[j] = HS_SUCCESS;
3981 : }
3982 : }
3983 :
3984 0 : if (i==0)
3985 0 : break;
3986 0 : }
3987 : }
3988 :
3989 0 : return HS_SUCCESS;
3990 0 : }
3991 :
3992 0 : int SchemaHistoryBase::hs_read(time_t start_time, time_t end_time, time_t interval,
3993 : int num_var,
3994 : const char* const event_name[], const char* const var_name[], const int var_index[],
3995 : int num_entries[],
3996 : time_t* time_buffer[], double* data_buffer[],
3997 : int st[])
3998 : {
3999 : int status;
4000 :
4001 0 : ReadBuffer** buffer = new ReadBuffer*[num_var];
4002 0 : MidasHistoryBufferInterface** bi = new MidasHistoryBufferInterface*[num_var];
4003 :
4004 0 : for (int i=0; i<num_var; i++) {
4005 0 : buffer[i] = new ReadBuffer(start_time, end_time, interval);
4006 0 : bi[i] = buffer[i];
4007 :
4008 : // make sure outputs are initialized to something sane
4009 0 : if (num_entries)
4010 0 : num_entries[i] = 0;
4011 0 : if (time_buffer)
4012 0 : time_buffer[i] = NULL;
4013 0 : if (data_buffer)
4014 0 : data_buffer[i] = NULL;
4015 0 : if (st)
4016 0 : st[i] = 0;
4017 :
4018 0 : if (num_entries)
4019 0 : buffer[i]->fNumEntries = &num_entries[i];
4020 0 : if (time_buffer)
4021 0 : buffer[i]->fTimeBuffer = &time_buffer[i];
4022 0 : if (data_buffer)
4023 0 : buffer[i]->fDataBuffer = &data_buffer[i];
4024 : }
4025 :
4026 0 : status = hs_read_buffer(start_time, end_time,
4027 : num_var, event_name, var_name, var_index,
4028 : bi, st);
4029 :
4030 0 : for (int i=0; i<num_var; i++) {
4031 0 : buffer[i]->Finish();
4032 0 : delete buffer[i];
4033 : }
4034 :
4035 0 : delete[] buffer;
4036 0 : delete[] bi;
4037 :
4038 0 : return status;
4039 : }
4040 :
4041 0 : int SchemaHistoryBase::hs_read_binned(time_t start_time, time_t end_time, int num_bins,
4042 : int num_var, const char* const event_name[], const char* const var_name[], const int var_index[],
4043 : int num_entries[],
4044 : int* count_bins[], double* mean_bins[], double* rms_bins[], double* min_bins[], double* max_bins[],
4045 : time_t* bins_first_time[], double* bins_first_value[],
4046 : time_t* bins_last_time[], double* bins_last_value[],
4047 : time_t last_time[], double last_value[],
4048 : int st[])
4049 : {
4050 : int status;
4051 :
4052 0 : MidasHistoryBinnedBuffer** buffer = new MidasHistoryBinnedBuffer*[num_var];
4053 0 : MidasHistoryBufferInterface** xbuffer = new MidasHistoryBufferInterface*[num_var];
4054 :
4055 0 : for (int i=0; i<num_var; i++) {
4056 0 : buffer[i] = new MidasHistoryBinnedBuffer(start_time, end_time, num_bins);
4057 0 : xbuffer[i] = buffer[i];
4058 :
4059 0 : if (count_bins)
4060 0 : buffer[i]->fCount = count_bins[i];
4061 0 : if (mean_bins)
4062 0 : buffer[i]->fMean = mean_bins[i];
4063 0 : if (rms_bins)
4064 0 : buffer[i]->fRms = rms_bins[i];
4065 0 : if (min_bins)
4066 0 : buffer[i]->fMin = min_bins[i];
4067 0 : if (max_bins)
4068 0 : buffer[i]->fMax = max_bins[i];
4069 0 : if (bins_first_time)
4070 0 : buffer[i]->fBinsFirstTime = bins_first_time[i];
4071 0 : if (bins_first_value)
4072 0 : buffer[i]->fBinsFirstValue = bins_first_value[i];
4073 0 : if (bins_last_time)
4074 0 : buffer[i]->fBinsLastTime = bins_last_time[i];
4075 0 : if (bins_last_value)
4076 0 : buffer[i]->fBinsLastValue = bins_last_value[i];
4077 0 : if (last_time)
4078 0 : buffer[i]->fLastTimePtr = &last_time[i];
4079 0 : if (last_value)
4080 0 : buffer[i]->fLastValuePtr = &last_value[i];
4081 :
4082 0 : buffer[i]->Start();
4083 : }
4084 :
4085 0 : status = hs_read_buffer(start_time, end_time,
4086 : num_var, event_name, var_name, var_index,
4087 : xbuffer,
4088 : st);
4089 :
4090 0 : for (int i=0; i<num_var; i++) {
4091 0 : buffer[i]->Finish();
4092 0 : if (num_entries)
4093 0 : num_entries[i] = buffer[i]->fNumEntries;
4094 : if (0) {
4095 : for (int j=0; j<num_bins; j++) {
4096 : printf("var %d bin %d count %d, first %s last %s value first %f last %f\n", i, j, count_bins[i][j], TimeToString(bins_first_time[i][j]).c_str(), TimeToString(bins_last_time[i][j]).c_str(), bins_first_value[i][j], bins_last_value[i][j]);
4097 : }
4098 : }
4099 0 : delete buffer[i];
4100 : }
4101 :
4102 0 : delete[] buffer;
4103 0 : delete[] xbuffer;
4104 :
4105 0 : return status;
4106 : }
4107 :
4108 : ////////////////////////////////////////////////////////
4109 : // SQL schema //
4110 : ////////////////////////////////////////////////////////
4111 :
4112 0 : int HsSqlSchema::close_transaction()
4113 : {
4114 0 : if (!fSql->IsConnected()) {
4115 0 : return HS_SUCCESS;
4116 : }
4117 :
4118 0 : int status = HS_SUCCESS;
4119 0 : if (get_transaction_count() > 0) {
4120 0 : status = fSql->CommitTransaction(fTableName.c_str());
4121 0 : reset_transaction_count();
4122 : }
4123 0 : return status;
4124 : }
4125 :
4126 0 : int HsSchema::match_event_var(const char* event_name, const char* var_name, const int var_index)
4127 : {
4128 0 : if (!MatchEventName(this->fEventName.c_str(), event_name))
4129 0 : return -1;
4130 :
4131 0 : for (size_t j=0; j<this->fVariables.size(); j++) {
4132 0 : if (MatchTagName(this->fVariables[j].name.c_str(), this->fVariables[j].n_data, var_name, var_index)) {
4133 : // Second clause in if() is case where MatchTagName used the "alternate tag name".
4134 : // E.g. our variable name is "IM05[3]" (n_data=1), but we're looking for var_name="IM05" and var_index=3.
4135 0 : if (var_index < this->fVariables[j].n_data || (this->fVariables[j].n_data == 1 && this->fVariables[j].name.find("[") != std::string::npos)) {
4136 0 : return j;
4137 : }
4138 : }
4139 : }
4140 :
4141 0 : return -1;
4142 : }
4143 :
4144 0 : int HsSqlSchema::match_event_var(const char* event_name, const char* var_name, const int var_index)
4145 : {
4146 0 : if (event_name_cmp(this->fTableName, event_name)==0) {
4147 0 : for (size_t j=0; j<this->fVariables.size(); j++) {
4148 0 : if (var_name_cmp(this->fColumnNames[j], var_name)==0)
4149 0 : return j;
4150 : }
4151 : }
4152 :
4153 0 : return HsSchema::match_event_var(event_name, var_name, var_index);
4154 : }
4155 :
4156 0 : static HsSqlSchema* NewSqlSchema(HsSchemaVector* sv, const char* table_name, time_t t)
4157 : {
4158 0 : time_t tt = 0;
4159 0 : int j=-1;
4160 0 : int jjx=-1; // remember oldest schema
4161 0 : time_t ttx = 0;
4162 0 : for (size_t i=0; i<sv->size(); i++) {
4163 0 : HsSqlSchema* s = (HsSqlSchema*)(*sv)[i];
4164 0 : if (s->fTableName != table_name)
4165 0 : continue;
4166 :
4167 0 : if (s->fTimeFrom == t) {
4168 0 : return s;
4169 : }
4170 :
4171 : // remember the last schema before time t
4172 0 : if (s->fTimeFrom < t) {
4173 0 : if (s->fTimeFrom > tt) {
4174 0 : tt = s->fTimeFrom;
4175 0 : j = i;
4176 : }
4177 : }
4178 :
4179 0 : if (jjx < 0) {
4180 0 : jjx = i;
4181 0 : ttx = s->fTimeFrom;
4182 : }
4183 :
4184 0 : if (s->fTimeFrom < ttx) {
4185 0 : jjx = i;
4186 0 : ttx = s->fTimeFrom;
4187 : }
4188 :
4189 : //printf("table_name [%s], t=%s, i=%d, j=%d %s, tt=%s, dt is %d\n", table_name, TimeToString(t).c_str(), i, j, TimeToString(s->fTimeFrom).c_str(), TimeToString(tt).c_str(), (int)(s->fTimeFrom-t));
4190 : }
4191 :
4192 : //printf("NewSqlSchema: will copy schema j=%d, tt=%d at time %d\n", j, tt, t);
4193 :
4194 : //printf("cloned schema at time %s: ", TimeToString(t).c_str());
4195 : //(*sv)[j]->print(false);
4196 :
4197 : //printf("schema before:\n");
4198 : //sv->print(false);
4199 :
4200 0 : if (j >= 0) {
4201 0 : HsSqlSchema* s = new HsSqlSchema;
4202 0 : *s = *(HsSqlSchema*)(*sv)[j]; // make a copy
4203 0 : s->fTimeFrom = t;
4204 0 : sv->add(s);
4205 :
4206 : //printf("schema after:\n");
4207 : //sv->print(false);
4208 :
4209 0 : return s;
4210 : }
4211 :
4212 0 : if (jjx >= 0) {
4213 0 : cm_msg(MERROR, "NewSqlSchema", "Error: Unexpected ordering of schema for table \'%s\', good luck!", table_name);
4214 :
4215 0 : HsSqlSchema* s = new HsSqlSchema;
4216 0 : *s = *(HsSqlSchema*)(*sv)[jjx]; // make a copy
4217 0 : s->fTimeFrom = t;
4218 0 : s->fTimeTo = ttx;
4219 0 : sv->add(s);
4220 :
4221 : //printf("schema after:\n");
4222 : //sv->print(false);
4223 :
4224 0 : return s;
4225 : }
4226 :
4227 0 : cm_msg(MERROR, "NewSqlSchema", "Error: Cannot clone schema for table \'%s\', good luck!", table_name);
4228 0 : return NULL;
4229 : }
4230 :
4231 0 : void HsSqlSchema::remove_inactive_columns()
4232 : {
4233 0 : assert(fVariables.size() == fColumnInactive.size());
4234 0 : assert(fVariables.size() == fColumnNames.size());
4235 0 : assert(fVariables.size() == fColumnTypes.size());
4236 0 : assert(fVariables.size() == fOffsets.size());
4237 :
4238 0 : size_t count_active = 0;
4239 0 : size_t count_inactive = 0;
4240 :
4241 0 : for (size_t i=0; i<fColumnInactive.size(); i++) {
4242 0 : if (fColumnInactive[i])
4243 0 : count_inactive += 1;
4244 : else
4245 0 : count_active += 1;
4246 : }
4247 :
4248 : //printf("remove_inactive_columns: enter! count_active: %zu, count_inactive: %zu\n", count_active, count_inactive);
4249 : //print();
4250 :
4251 0 : if (count_inactive > 0) {
4252 0 : size_t j=0;
4253 :
4254 0 : for (size_t i=0; i<fColumnInactive.size(); i++) {
4255 0 : if (fColumnInactive[i]) {
4256 : // skip this entry
4257 : } else {
4258 0 : if (j != i) {
4259 0 : fVariables[j] = fVariables[i];
4260 0 : fColumnInactive[j] = fColumnInactive[i];
4261 0 : fColumnNames[j] = fColumnNames[i];
4262 0 : fColumnTypes[j] = fColumnTypes[i];
4263 0 : fOffsets[j] = fOffsets[i];
4264 : }
4265 0 : j++;
4266 : }
4267 : }
4268 :
4269 : //print();
4270 : //printf("%zu %zu\n", j, count_active);
4271 :
4272 0 : assert(j == count_active);
4273 :
4274 : //print();
4275 :
4276 0 : fVariables.resize(count_active);
4277 0 : fColumnInactive.resize(count_active);
4278 0 : fColumnNames.resize(count_active);
4279 0 : fColumnTypes.resize(count_active);
4280 0 : fOffsets.resize(count_active);
4281 :
4282 0 : assert(fVariables.size() == fColumnInactive.size());
4283 0 : assert(fVariables.size() == fColumnNames.size());
4284 0 : assert(fVariables.size() == fColumnTypes.size());
4285 0 : assert(fVariables.size() == fOffsets.size());
4286 :
4287 : //printf("remove_inactice_columns: exit!\n");
4288 : //print();
4289 : }
4290 0 : }
4291 :
4292 0 : int HsSqlSchema::write_event(const time_t t, const char* data, const size_t data_size)
4293 : {
4294 0 : HsSqlSchema* s = this;
4295 :
4296 0 : assert(s->fVariables.size() == s->fColumnInactive.size());
4297 0 : assert(s->fVariables.size() == s->fColumnNames.size());
4298 0 : assert(s->fVariables.size() == s->fColumnTypes.size());
4299 0 : assert(s->fVariables.size() == s->fOffsets.size());
4300 :
4301 0 : std::string tags;
4302 0 : std::string values;
4303 :
4304 0 : for (size_t i=0; i<s->fVariables.size(); i++) {
4305 : // NB: inactive columns should have been removed from the schema. K.O.
4306 :
4307 0 : if (s->fColumnInactive[i]) {
4308 0 : cm_msg(MERROR, "HsSqlSchema::write_event", "Internal error, unexpected inactive column %zu", i);
4309 0 : cm_msg_flush_buffer();
4310 0 : return HS_FILE_ERROR;
4311 : }
4312 :
4313 0 : int type = s->fVariables[i].type;
4314 0 : int n_data = s->fVariables[i].n_data;
4315 0 : int offset = s->fOffsets[i];
4316 0 : const char* column_name = s->fColumnNames[i].c_str();
4317 :
4318 0 : if (offset < 0) {
4319 0 : cm_msg(MERROR, "HsSqlSchema::write_event", "Internal error, unexpected negative offset %d for column %zu", offset, i);
4320 0 : cm_msg_flush_buffer();
4321 0 : return HS_FILE_ERROR;
4322 : }
4323 :
4324 0 : assert(n_data == 1);
4325 0 : assert(strlen(column_name) > 0);
4326 0 : assert(offset >= 0);
4327 0 : assert((size_t)offset < data_size);
4328 :
4329 0 : void* ptr = (void*)(data+offset);
4330 :
4331 0 : tags += ", ";
4332 0 : tags += fSql->QuoteId(column_name);
4333 :
4334 0 : values += ", ";
4335 :
4336 : char buf[1024];
4337 0 : int j=0;
4338 :
4339 0 : switch (type) {
4340 0 : default:
4341 0 : sprintf(buf, "unknownType%d", type);
4342 0 : break;
4343 0 : case TID_BYTE:
4344 0 : sprintf(buf, "%u",((unsigned char *)ptr)[j]);
4345 0 : break;
4346 0 : case TID_SBYTE:
4347 0 : sprintf(buf, "%d",((signed char*)ptr)[j]);
4348 0 : break;
4349 0 : case TID_CHAR:
4350 : // FIXME: quotes
4351 0 : sprintf(buf, "\'%c\'",((char*)ptr)[j]);
4352 0 : break;
4353 0 : case TID_WORD:
4354 0 : sprintf(buf, "%u",((unsigned short *)ptr)[j]);
4355 0 : break;
4356 0 : case TID_SHORT:
4357 0 : sprintf(buf, "%d",((short *)ptr)[j]);
4358 0 : break;
4359 0 : case TID_DWORD:
4360 0 : sprintf(buf, "%u",((unsigned int *)ptr)[j]);
4361 0 : break;
4362 0 : case TID_INT:
4363 0 : sprintf(buf, "%d",((int *)ptr)[j]);
4364 0 : break;
4365 0 : case TID_BOOL:
4366 0 : sprintf(buf, "%u",((unsigned int *)ptr)[j]);
4367 0 : break;
4368 0 : case TID_FLOAT:
4369 : // FIXME: quotes
4370 0 : sprintf(buf, "\'%.8g\'",((float*)ptr)[j]);
4371 0 : break;
4372 0 : case TID_DOUBLE:
4373 : // FIXME: quotes
4374 0 : sprintf(buf, "\'%.16g\'",((double*)ptr)[j]);
4375 0 : break;
4376 : }
4377 :
4378 0 : values += buf;
4379 : }
4380 :
4381 : // 2001-02-16 20:38:40.1
4382 : struct tm tms;
4383 0 : localtime_r(&t, &tms); // somebody must call tzset() before this.
4384 : char buf[1024];
4385 0 : strftime(buf, sizeof(buf)-1, "%Y-%m-%d %H:%M:%S.0", &tms);
4386 :
4387 0 : std::string cmd;
4388 0 : cmd = "INSERT INTO ";
4389 0 : cmd += fSql->QuoteId(s->fTableName.c_str());
4390 0 : cmd += " (_t_time, _i_time";
4391 0 : cmd += tags;
4392 0 : cmd += ") VALUES (";
4393 0 : cmd += fSql->QuoteString(buf);
4394 0 : cmd += ", ";
4395 0 : cmd += fSql->QuoteString(TimeToString(t).c_str());
4396 0 : cmd += "";
4397 0 : cmd += values;
4398 0 : cmd += ");";
4399 :
4400 0 : if (fSql->IsConnected()) {
4401 0 : if (s->get_transaction_count() == 0)
4402 0 : fSql->OpenTransaction(s->fTableName.c_str());
4403 :
4404 0 : s->increment_transaction_count();
4405 :
4406 0 : int status = fSql->Exec(s->fTableName.c_str(), cmd.c_str());
4407 :
4408 : // mh2sql who does not call hs_flush_buffers()
4409 : // so we should flush the transaction by hand
4410 : // some SQL engines have limited transaction buffers... K.O.
4411 0 : if (s->get_transaction_count() > 100000) {
4412 : //printf("flush table %s\n", table_name);
4413 0 : fSql->CommitTransaction(s->fTableName.c_str());
4414 0 : s->reset_transaction_count();
4415 : }
4416 :
4417 0 : if (status != DB_SUCCESS) {
4418 0 : return status;
4419 : }
4420 : } else {
4421 0 : int status = fSql->ExecDisconnected(s->fTableName.c_str(), cmd.c_str());
4422 0 : if (status != DB_SUCCESS) {
4423 0 : return status;
4424 : }
4425 : }
4426 :
4427 0 : return HS_SUCCESS;
4428 0 : }
4429 :
4430 0 : int HsSqlSchema::read_last_written(const time_t timestamp,
4431 : const int debug,
4432 : time_t* last_written)
4433 : {
4434 0 : if (debug)
4435 0 : printf("SqlHistory::read_last_written: table [%s], timestamp %s\n", fTableName.c_str(), TimeToString(timestamp).c_str());
4436 :
4437 0 : std::string cmd;
4438 0 : cmd += "SELECT _i_time FROM ";
4439 0 : cmd += fSql->QuoteId(fTableName.c_str());
4440 0 : cmd += " WHERE _i_time < ";
4441 0 : cmd += TimeToString(timestamp);
4442 0 : cmd += " ORDER BY _i_time DESC LIMIT 2;";
4443 :
4444 0 : int status = fSql->Prepare(fTableName.c_str(), cmd.c_str());
4445 :
4446 0 : if (status != DB_SUCCESS)
4447 0 : return status;
4448 :
4449 0 : time_t lw = 0;
4450 :
4451 : /* Loop through the rows in the result-set */
4452 :
4453 : while (1) {
4454 0 : status = fSql->Step();
4455 0 : if (status != DB_SUCCESS)
4456 0 : break;
4457 :
4458 0 : time_t t = fSql->GetTime(0);
4459 :
4460 0 : if (t >= timestamp)
4461 0 : continue;
4462 :
4463 0 : if (t > lw)
4464 0 : lw = t;
4465 0 : }
4466 :
4467 0 : fSql->Finalize();
4468 :
4469 0 : *last_written = lw;
4470 :
4471 0 : if (debug)
4472 0 : printf("SqlHistory::read_last_written: table [%s], timestamp %s, last_written %s\n", fTableName.c_str(), TimeToString(timestamp).c_str(), TimeToString(lw).c_str());
4473 :
4474 0 : return HS_SUCCESS;
4475 0 : }
4476 :
4477 0 : int HsSqlSchema::read_data(const time_t start_time,
4478 : const time_t end_time,
4479 : const int num_var, const std::vector<int>& var_schema_index, const int var_index[],
4480 : const int debug,
4481 : std::vector<time_t>& last_time,
4482 : MidasHistoryBufferInterface* buffer[])
4483 : {
4484 0 : bool bad_last_time = false;
4485 :
4486 0 : if (debug)
4487 0 : printf("SqlHistory::read_data: table [%s], start %s, end %s\n", fTableName.c_str(), TimeToString(start_time).c_str(), TimeToString(end_time).c_str());
4488 :
4489 0 : std::string collist;
4490 :
4491 0 : for (int i=0; i<num_var; i++) {
4492 0 : int j = var_schema_index[i];
4493 0 : if (j < 0)
4494 0 : continue;
4495 0 : if (collist.length() > 0)
4496 0 : collist += ", ";
4497 0 : collist += fSql->QuoteId(fColumnNames[j].c_str());
4498 : }
4499 :
4500 0 : std::string cmd;
4501 0 : cmd += "SELECT _i_time, ";
4502 0 : cmd += collist;
4503 0 : cmd += " FROM ";
4504 0 : cmd += fSql->QuoteId(fTableName.c_str());
4505 0 : cmd += " WHERE _i_time>=";
4506 0 : cmd += TimeToString(start_time);
4507 0 : cmd += " and _i_time<=";
4508 0 : cmd += TimeToString(end_time);
4509 0 : cmd += " ORDER BY _i_time;";
4510 :
4511 0 : int status = fSql->Prepare(fTableName.c_str(), cmd.c_str());
4512 :
4513 0 : if (status != DB_SUCCESS)
4514 0 : return HS_FILE_ERROR;
4515 :
4516 : /* Loop through the rows in the result-set */
4517 :
4518 0 : int count = 0;
4519 :
4520 : while (1) {
4521 0 : status = fSql->Step();
4522 0 : if (status != DB_SUCCESS)
4523 0 : break;
4524 :
4525 0 : count++;
4526 :
4527 0 : time_t t = fSql->GetTime(0);
4528 :
4529 0 : if (t < start_time || t > end_time)
4530 0 : continue;
4531 :
4532 0 : int k = 0;
4533 :
4534 0 : for (int i=0; i<num_var; i++) {
4535 0 : int j = var_schema_index[i];
4536 0 : if (j < 0)
4537 0 : continue;
4538 :
4539 0 : if (t < last_time[i]) { // protect against duplicate and non-monotonous data
4540 0 : bad_last_time = true;
4541 : } else {
4542 0 : double v = fSql->GetDouble(1+k);
4543 :
4544 : //printf("Column %d, index %d, Row %d, time %d, value %f\n", k, colindex[k], count, t, v);
4545 :
4546 0 : buffer[i]->Add(t, v);
4547 0 : last_time[i] = t;
4548 : }
4549 :
4550 0 : k++;
4551 : }
4552 0 : }
4553 :
4554 0 : fSql->Finalize();
4555 :
4556 0 : if (bad_last_time) {
4557 0 : cm_msg(MERROR, "SqlHistory::read_data", "Detected duplicate or non-monotonous data in table \"%s\" for start time %s and end time %s", fTableName.c_str(), TimeToString(start_time).c_str(), TimeToString(end_time).c_str());
4558 : }
4559 :
4560 0 : if (debug)
4561 0 : printf("SqlHistory::read_data: table [%s], start %s, end %s, read %d rows\n", fTableName.c_str(), TimeToString(start_time).c_str(), TimeToString(end_time).c_str(), count);
4562 :
4563 0 : return HS_SUCCESS;
4564 0 : }
4565 :
4566 0 : int HsSqlSchema::get_transaction_count() {
4567 0 : if (!fSql || fSql->fTransactionPerTable) {
4568 0 : return fTableTransactionCount;
4569 : } else {
4570 0 : return gfTransactionCount[fSql];
4571 : }
4572 : }
4573 :
4574 0 : void HsSqlSchema::reset_transaction_count() {
4575 0 : if (!fSql || fSql->fTransactionPerTable) {
4576 0 : fTableTransactionCount = 0;
4577 : } else {
4578 0 : gfTransactionCount[fSql] = 0;
4579 : }
4580 0 : }
4581 :
4582 0 : void HsSqlSchema::increment_transaction_count() {
4583 0 : if (!fSql || fSql->fTransactionPerTable) {
4584 0 : fTableTransactionCount++;
4585 : } else {
4586 0 : gfTransactionCount[fSql]++;
4587 : }
4588 0 : }
4589 :
4590 : ////////////////////////////////////////////////////////
4591 : // SQL history functions //
4592 : ////////////////////////////////////////////////////////
4593 :
4594 0 : static int StartSqlTransaction(SqlBase* sql, const char* table_name, bool* have_transaction)
4595 : {
4596 0 : if (*have_transaction)
4597 0 : return HS_SUCCESS;
4598 :
4599 0 : int status = sql->OpenTransaction(table_name);
4600 0 : if (status != DB_SUCCESS)
4601 0 : return HS_FILE_ERROR;
4602 :
4603 0 : *have_transaction = true;
4604 0 : return HS_SUCCESS;
4605 : }
4606 :
4607 0 : static int CreateSqlTable(SqlBase* sql, const char* table_name, bool* have_transaction, bool set_default_timestamp = false)
4608 : {
4609 : int status;
4610 :
4611 0 : status = StartSqlTransaction(sql, table_name, have_transaction);
4612 0 : if (status != DB_SUCCESS)
4613 0 : return HS_FILE_ERROR;
4614 :
4615 0 : std::string cmd;
4616 :
4617 0 : cmd = "CREATE TABLE ";
4618 0 : cmd += sql->QuoteId(table_name);
4619 0 : if (set_default_timestamp) {
4620 0 : cmd += " (_t_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, _i_time INTEGER NOT NULL DEFAULT 0);";
4621 : } else {
4622 0 : cmd += " (_t_time TIMESTAMP NOT NULL, _i_time INTEGER NOT NULL);";
4623 : }
4624 :
4625 0 : status = sql->Exec(table_name, cmd.c_str());
4626 :
4627 :
4628 0 : if (status == DB_KEY_EXIST) {
4629 0 : cm_msg(MINFO, "CreateSqlTable", "Adding SQL table \"%s\", but it already exists", table_name);
4630 0 : cm_msg_flush_buffer();
4631 0 : return status;
4632 : }
4633 :
4634 0 : if (status != DB_SUCCESS) {
4635 0 : cm_msg(MINFO, "CreateSqlTable", "Adding SQL table \"%s\", error status %d", table_name, status);
4636 0 : cm_msg_flush_buffer();
4637 0 : return HS_FILE_ERROR;
4638 : }
4639 :
4640 0 : cm_msg(MINFO, "CreateSqlTable", "Adding SQL table \"%s\"", table_name);
4641 0 : cm_msg_flush_buffer();
4642 :
4643 0 : std::string i_index_name;
4644 0 : i_index_name = table_name;
4645 0 : i_index_name += "_i_time_index";
4646 :
4647 0 : std::string t_index_name;
4648 0 : t_index_name = table_name;
4649 0 : t_index_name += "_t_time_index";
4650 :
4651 0 : cmd = "CREATE INDEX ";
4652 0 : cmd += sql->QuoteId(i_index_name.c_str());
4653 0 : cmd += " ON ";
4654 0 : cmd += sql->QuoteId(table_name);
4655 0 : cmd += " (_i_time ASC);";
4656 :
4657 0 : status = sql->Exec(table_name, cmd.c_str());
4658 0 : if (status != DB_SUCCESS)
4659 0 : return HS_FILE_ERROR;
4660 :
4661 0 : cmd = "CREATE INDEX ";
4662 0 : cmd += sql->QuoteId(t_index_name.c_str());
4663 0 : cmd += " ON ";
4664 0 : cmd += sql->QuoteId(table_name);
4665 0 : cmd += " (_t_time);";
4666 :
4667 0 : status = sql->Exec(table_name, cmd.c_str());
4668 0 : if (status != DB_SUCCESS)
4669 0 : return HS_FILE_ERROR;
4670 :
4671 0 : return status;
4672 0 : }
4673 :
4674 0 : static int CreateSqlHyperTable(SqlBase* sql, const char* table_name, bool* have_transaction) {
4675 : int status;
4676 :
4677 0 : status = StartSqlTransaction(sql, table_name, have_transaction);
4678 0 : if (status != DB_SUCCESS)
4679 0 : return HS_FILE_ERROR;
4680 :
4681 0 : std::string cmd;
4682 :
4683 0 : cmd = "CREATE TABLE ";
4684 0 : cmd += sql->QuoteId(table_name);
4685 0 : cmd += " (_t_time TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, _i_time INTEGER NOT NULL DEFAULT 0);";
4686 :
4687 0 : status = sql->Exec(table_name, cmd.c_str());
4688 :
4689 0 : if (status == DB_KEY_EXIST) {
4690 0 : cm_msg(MINFO, "CreateSqlHyperTable", "Adding SQL table \"%s\", but it already exists", table_name);
4691 0 : cm_msg_flush_buffer();
4692 0 : return status;
4693 : }
4694 :
4695 0 : if (status != DB_SUCCESS) {
4696 0 : cm_msg(MINFO, "CreateSqlHyperTable", "Adding SQL table \"%s\", error status %d", table_name, status);
4697 0 : cm_msg_flush_buffer();
4698 0 : return HS_FILE_ERROR;
4699 : }
4700 :
4701 0 : cm_msg(MINFO, "CreateSqlHyperTable", "Adding SQL table \"%s\"", table_name);
4702 0 : cm_msg_flush_buffer();
4703 :
4704 0 : cmd = "SELECT create_hypertable(";
4705 0 : cmd += sql->QuoteString(table_name);
4706 0 : cmd += ", '_t_time');";
4707 :
4708 : // convert regular table to hypertable
4709 0 : status = sql->Exec(table_name, cmd.c_str());
4710 :
4711 0 : if (status != DB_SUCCESS) {
4712 0 : cm_msg(MINFO, "CreateSqlHyperTable", "Converting SQL table to hypertable \"%s\", error status %d", table_name, status);
4713 0 : cm_msg_flush_buffer();
4714 0 : return HS_FILE_ERROR;
4715 : }
4716 :
4717 0 : std::string i_index_name;
4718 0 : i_index_name = table_name;
4719 0 : i_index_name += "_i_time_index";
4720 :
4721 0 : std::string t_index_name;
4722 0 : t_index_name = table_name;
4723 0 : t_index_name += "_t_time_index";
4724 :
4725 0 : cmd = "CREATE INDEX ";
4726 0 : cmd += sql->QuoteId(i_index_name.c_str());
4727 0 : cmd += " ON ";
4728 0 : cmd += sql->QuoteId(table_name);
4729 0 : cmd += " (_i_time ASC);";
4730 :
4731 0 : status = sql->Exec(table_name, cmd.c_str());
4732 0 : if (status != DB_SUCCESS)
4733 0 : return HS_FILE_ERROR;
4734 :
4735 0 : cmd = "CREATE INDEX ";
4736 0 : cmd += sql->QuoteId(t_index_name.c_str());
4737 0 : cmd += " ON ";
4738 0 : cmd += sql->QuoteId(table_name);
4739 0 : cmd += " (_t_time);";
4740 :
4741 0 : status = sql->Exec(table_name, cmd.c_str());
4742 0 : if (status != DB_SUCCESS)
4743 0 : return HS_FILE_ERROR;
4744 :
4745 0 : return status;
4746 0 : }
4747 :
4748 0 : static int CreateSqlColumn(SqlBase* sql, const char* table_name, const char* column_name, const char* column_type, bool* have_transaction, int debug)
4749 : {
4750 0 : if (debug)
4751 0 : printf("CreateSqlColumn: table [%s], column [%s], type [%s]\n", table_name, column_name, column_type);
4752 :
4753 0 : int status = StartSqlTransaction(sql, table_name, have_transaction);
4754 0 : if (status != HS_SUCCESS)
4755 0 : return status;
4756 :
4757 0 : std::string cmd;
4758 0 : cmd = "ALTER TABLE ";
4759 0 : cmd += sql->QuoteId(table_name);
4760 0 : cmd += " ADD COLUMN ";
4761 0 : cmd += sql->QuoteId(column_name);
4762 0 : cmd += " ";
4763 0 : cmd += column_type;
4764 0 : cmd += ";";
4765 :
4766 0 : status = sql->Exec(table_name, cmd.c_str());
4767 :
4768 0 : cm_msg(MINFO, "CreateSqlColumn", "Adding column \"%s\" to SQL table \"%s\", status %d", column_name, table_name, status);
4769 0 : cm_msg_flush_buffer();
4770 :
4771 0 : return status;
4772 0 : }
4773 :
4774 : ////////////////////////////////////////////////////////
4775 : // SQL history base classes //
4776 : ////////////////////////////////////////////////////////
4777 :
4778 : class SqlHistoryBase: public SchemaHistoryBase
4779 : {
4780 : public:
4781 : SqlBase *fSql;
4782 :
4783 0 : SqlHistoryBase() // ctor
4784 0 : {
4785 0 : fSql = NULL;
4786 0 : hs_clear_cache();
4787 0 : }
4788 :
4789 0 : virtual ~SqlHistoryBase() // dtor
4790 0 : {
4791 0 : hs_disconnect();
4792 0 : if (fSql)
4793 0 : delete fSql;
4794 0 : fSql = NULL;
4795 0 : }
4796 :
4797 0 : int hs_set_debug(int debug)
4798 : {
4799 0 : if (fSql)
4800 0 : fSql->fDebug = debug;
4801 0 : return SchemaHistoryBase::hs_set_debug(debug);
4802 : }
4803 :
4804 : int hs_connect(const char* connect_string);
4805 : int hs_disconnect();
4806 : HsSchema* new_event(const char* event_name, time_t timestamp, int ntags, const TAG tags[]);
4807 : int read_schema(HsSchemaVector* sv, const char* event_name, const time_t timestamp);
4808 0 : HsSchema* maybe_reopen(const char* event_name, time_t timestamp, HsSchema* s) { return s; };
4809 :
4810 :
4811 : protected:
4812 : virtual int read_table_and_event_names(HsSchemaVector *sv) = 0;
4813 : virtual int read_column_names(HsSchemaVector *sv, const char* table_name, const char* event_name) = 0;
4814 : virtual int create_table(HsSchemaVector* sv, const char* event_name, time_t timestamp) = 0;
4815 : virtual int update_column(const char* event_name, const char* table_name, const char* column_name, const char* column_type, const char* tag_name, const char* tag_type, const time_t timestamp, bool active, bool* have_transaction) = 0;
4816 :
4817 : int update_schema(HsSqlSchema* s, const time_t timestamp, const int ntags, const TAG tags[], bool write_enable);
4818 : int update_schema1(HsSqlSchema* s, const time_t timestamp, const int ntags, const TAG tags[], bool write_enable, bool* have_transaction);
4819 : };
4820 :
4821 0 : int SqlHistoryBase::hs_connect(const char* connect_string)
4822 : {
4823 0 : if (fDebug)
4824 0 : printf("hs_connect [%s]!\n", connect_string);
4825 :
4826 0 : assert(fSql);
4827 :
4828 0 : if (fSql->IsConnected())
4829 0 : if (strcmp(fConnectString.c_str(), connect_string) == 0)
4830 0 : return HS_SUCCESS;
4831 :
4832 0 : hs_disconnect();
4833 :
4834 0 : if (!connect_string || strlen(connect_string) < 1) {
4835 : // FIXME: should use "logger dir" or some such default, that code should be in hs_get_history(), not here
4836 0 : connect_string = ".";
4837 : }
4838 :
4839 0 : fConnectString = connect_string;
4840 :
4841 0 : if (fDebug)
4842 0 : printf("hs_connect: connecting to SQL database \'%s\'\n", fConnectString.c_str());
4843 :
4844 0 : int status = fSql->Connect(fConnectString.c_str());
4845 0 : if (status != DB_SUCCESS)
4846 0 : return status;
4847 :
4848 0 : return HS_SUCCESS;
4849 : }
4850 :
4851 0 : int SqlHistoryBase::hs_disconnect()
4852 : {
4853 0 : if (fDebug)
4854 0 : printf("hs_disconnect!\n");
4855 :
4856 0 : hs_flush_buffers();
4857 :
4858 0 : fSql->Disconnect();
4859 :
4860 0 : hs_clear_cache();
4861 :
4862 0 : return HS_SUCCESS;
4863 : }
4864 :
4865 0 : HsSchema* SqlHistoryBase::new_event(const char* event_name, time_t timestamp, int ntags, const TAG tags[])
4866 : {
4867 0 : if (fDebug)
4868 0 : printf("SqlHistory::new_event: event [%s], timestamp %s, ntags %d\n", event_name, TimeToString(timestamp).c_str(), ntags);
4869 :
4870 : int status;
4871 :
4872 0 : if (fWriterSchema.size() == 0) {
4873 0 : status = read_table_and_event_names(&fWriterSchema);
4874 0 : if (status != HS_SUCCESS)
4875 0 : return NULL;
4876 : }
4877 :
4878 0 : HsSqlSchema* s = (HsSqlSchema*)fWriterSchema.find_event(event_name, timestamp);
4879 :
4880 : // schema does not exist, the SQL tables probably do not exist yet
4881 :
4882 0 : if (!s) {
4883 0 : status = create_table(&fWriterSchema, event_name, timestamp);
4884 0 : if (status != HS_SUCCESS)
4885 0 : return NULL;
4886 :
4887 0 : s = (HsSqlSchema*)fWriterSchema.find_event(event_name, timestamp);
4888 :
4889 0 : if (!s) {
4890 0 : cm_msg(MERROR, "SqlHistory::new_event", "Error: Cannot create schema for event \'%s\', see previous messages", event_name);
4891 0 : fWriterSchema.find_event(event_name, timestamp, 1);
4892 0 : return NULL;
4893 : }
4894 : }
4895 :
4896 0 : assert(s != NULL);
4897 :
4898 0 : status = read_column_names(&fWriterSchema, s->fTableName.c_str(), s->fEventName.c_str());
4899 0 : if (status != HS_SUCCESS)
4900 0 : return NULL;
4901 :
4902 0 : s = (HsSqlSchema*)fWriterSchema.find_event(event_name, timestamp);
4903 :
4904 0 : if (!s) {
4905 0 : cm_msg(MERROR, "SqlHistory::new_event", "Error: Cannot update schema database for event \'%s\', see previous messages", event_name);
4906 0 : return NULL;
4907 : }
4908 :
4909 0 : if (0||fDebug) {
4910 0 : printf("SqlHistory::new_event: schema for [%s] is %p\n", event_name, s);
4911 0 : if (s)
4912 0 : s->print();
4913 : }
4914 :
4915 0 : status = update_schema(s, timestamp, ntags, tags, true);
4916 0 : if (status != HS_SUCCESS) {
4917 0 : cm_msg(MERROR, "SqlHistory::new_event", "Error: Cannot create schema for event \'%s\', see previous messages", event_name);
4918 0 : return NULL;
4919 : }
4920 :
4921 0 : status = read_column_names(&fWriterSchema, s->fTableName.c_str(), s->fEventName.c_str());
4922 0 : if (status != HS_SUCCESS)
4923 0 : return NULL;
4924 :
4925 0 : s = (HsSqlSchema*)fWriterSchema.find_event(event_name, timestamp);
4926 :
4927 0 : if (!s) {
4928 0 : cm_msg(MERROR, "SqlHistory::new_event", "Error: Cannot update schema database for event \'%s\', see previous messages", event_name);
4929 0 : return NULL;
4930 : }
4931 :
4932 0 : if (0||fDebug) {
4933 0 : printf("SqlHistory::new_event: schema for [%s] is %p\n", event_name, s);
4934 0 : if (s)
4935 0 : s->print();
4936 : }
4937 :
4938 : // last call to UpdateMysqlSchema with "false" will check that new schema matches the new tags
4939 :
4940 0 : status = update_schema(s, timestamp, ntags, tags, false);
4941 0 : if (status != HS_SUCCESS) {
4942 0 : cm_msg(MERROR, "SqlHistory::new_event", "Error: Cannot create schema for event \'%s\', see previous messages", event_name);
4943 : //fDebug = 1;
4944 : //update_schema(s, timestamp, ntags, tags, false);
4945 : //abort();
4946 0 : return NULL;
4947 : }
4948 :
4949 0 : HsSqlSchema* e = new HsSqlSchema();
4950 :
4951 0 : *e = *s; // make a copy of the schema
4952 :
4953 0 : return e;
4954 : }
4955 :
4956 0 : int SqlHistoryBase::read_schema(HsSchemaVector* sv, const char* event_name, const time_t timestamp)
4957 : {
4958 0 : if (fDebug)
4959 0 : printf("SqlHistory::read_schema: loading schema for event [%s] at time %s\n", event_name, TimeToString(timestamp).c_str());
4960 :
4961 : int status;
4962 :
4963 0 : if (sv->size() == 0) {
4964 0 : status = read_table_and_event_names(sv);
4965 0 : if (status != HS_SUCCESS)
4966 0 : return status;
4967 : }
4968 :
4969 : //sv->print(false);
4970 :
4971 0 : if (event_name == NULL)
4972 0 : return HS_SUCCESS;
4973 :
4974 0 : for (size_t i=0; i<sv->size(); i++) {
4975 0 : HsSqlSchema* h = (HsSqlSchema*)(*sv)[i];
4976 : // skip schema with already read column names
4977 0 : if (h->fVariables.size() > 0)
4978 0 : continue;
4979 : // skip schema with different name
4980 0 : if (!MatchEventName(h->fEventName.c_str(), event_name))
4981 0 : continue;
4982 :
4983 0 : size_t nn = sv->size();
4984 :
4985 0 : status = read_column_names(sv, h->fTableName.c_str(), h->fEventName.c_str());
4986 :
4987 : // if new schema was added, loop all over again
4988 0 : if (sv->size() != nn)
4989 0 : i=0;
4990 : }
4991 :
4992 : //sv->print(false);
4993 :
4994 0 : return HS_SUCCESS;
4995 : }
4996 :
4997 0 : int SqlHistoryBase::update_schema(HsSqlSchema* s, const time_t timestamp, const int ntags, const TAG tags[], bool write_enable)
4998 : {
4999 : int status;
5000 0 : bool have_transaction = false;
5001 :
5002 0 : status = update_schema1(s, timestamp, ntags, tags, write_enable, &have_transaction);
5003 :
5004 0 : if (have_transaction) {
5005 : int xstatus;
5006 :
5007 0 : if (status == HS_SUCCESS)
5008 0 : xstatus = fSql->CommitTransaction(s->fTableName.c_str());
5009 : else
5010 0 : xstatus = fSql->RollbackTransaction(s->fTableName.c_str());
5011 :
5012 0 : if (xstatus != DB_SUCCESS) {
5013 0 : return HS_FILE_ERROR;
5014 : }
5015 0 : have_transaction = false;
5016 : }
5017 :
5018 0 : return status;
5019 : }
5020 :
5021 0 : int SqlHistoryBase::update_schema1(HsSqlSchema* s, const time_t timestamp, const int ntags, const TAG tags[], bool write_enable, bool* have_transaction)
5022 : {
5023 : int status;
5024 :
5025 0 : if (fDebug)
5026 0 : printf("update_schema1\n");
5027 :
5028 : // check that compare schema with tags[]
5029 :
5030 0 : bool schema_ok = true;
5031 :
5032 0 : int offset = 0;
5033 0 : for (int i=0; i<ntags; i++) {
5034 0 : for (unsigned int j=0; j<tags[i].n_data; j++) {
5035 0 : int tagtype = tags[i].type;
5036 0 : std::string tagname = tags[i].name;
5037 0 : std::string maybe_colname = MidasNameToSqlName(tags[i].name);
5038 :
5039 0 : if (tags[i].n_data > 1) {
5040 : char s[256];
5041 0 : sprintf(s, "[%d]", j);
5042 0 : tagname += s;
5043 :
5044 0 : sprintf(s, "_%d", j);
5045 0 : maybe_colname += s;
5046 : }
5047 :
5048 0 : int count = 0;
5049 :
5050 0 : for (size_t j=0; j<s->fVariables.size(); j++) {
5051 : // NB: inactive columns will be reactivated or recreated by the if(count==0) branch. K.O.
5052 0 : if (s->fColumnInactive[j])
5053 0 : continue;
5054 0 : if (tagname == s->fVariables[j].name) {
5055 0 : if (s->fSql->TypesCompatible(tagtype, s->fColumnTypes[j].c_str())) {
5056 0 : if (count == 0) {
5057 0 : s->fOffsets[j] = offset;
5058 0 : offset += rpc_tid_size(tagtype);
5059 : }
5060 0 : count++;
5061 0 : if (count > 1) {
5062 0 : cm_msg(MERROR, "SqlHistory::update_schema", "Duplicate SQL column \'%s\' type \'%s\' in table \"%s\" with MIDAS type \'%s\' history event \"%s\" tag \"%s\"", s->fColumnNames[j].c_str(), s->fColumnTypes[j].c_str(), s->fTableName.c_str(), rpc_tid_name(tagtype), s->fEventName.c_str(), tagname.c_str());
5063 0 : cm_msg_flush_buffer();
5064 : }
5065 : } else {
5066 : // column with incompatible type, mark it as unused
5067 0 : schema_ok = false;
5068 0 : if (fDebug)
5069 0 : printf("Incompatible column!\n");
5070 0 : if (write_enable) {
5071 0 : cm_msg(MINFO, "SqlHistory::update_schema", "Deactivating SQL column \'%s\' type \'%s\' in table \"%s\" as incompatible with MIDAS type \'%s\' history event \"%s\" tag \"%s\"", s->fColumnNames[j].c_str(), s->fColumnTypes[j].c_str(), s->fTableName.c_str(), rpc_tid_name(tagtype), s->fEventName.c_str(), tagname.c_str());
5072 0 : cm_msg_flush_buffer();
5073 :
5074 0 : status = update_column(s->fEventName.c_str(), s->fTableName.c_str(), s->fColumnNames[j].c_str(), s->fColumnTypes[j].c_str(), s->fVariables[j].tag_name.c_str(), s->fVariables[i].tag_type.c_str(), timestamp, false, have_transaction);
5075 0 : if (status != HS_SUCCESS)
5076 0 : return status;
5077 : }
5078 : }
5079 : }
5080 : }
5081 :
5082 0 : if (count == 0) {
5083 : // tag does not have a corresponding column
5084 0 : schema_ok = false;
5085 0 : if (fDebug)
5086 0 : printf("No column for tag %s!\n", tagname.c_str());
5087 :
5088 0 : bool found_column = false;
5089 :
5090 0 : if (write_enable) {
5091 0 : for (size_t j=0; j<s->fVariables.size(); j++) {
5092 0 : if (tagname == s->fVariables[j].tag_name) {
5093 0 : bool typeok = s->fSql->TypesCompatible(tagtype, s->fColumnTypes[j].c_str());
5094 0 : if (typeok) {
5095 0 : cm_msg(MINFO, "SqlHistory::update_schema", "Reactivating SQL column \'%s\' type \'%s\' in table \"%s\" for history event \"%s\" tag \"%s\"", s->fColumnNames[j].c_str(), s->fColumnTypes[j].c_str(), s->fTableName.c_str(), s->fEventName.c_str(), tagname.c_str());
5096 0 : cm_msg_flush_buffer();
5097 :
5098 0 : status = update_column(s->fEventName.c_str(), s->fTableName.c_str(), s->fColumnNames[j].c_str(), s->fColumnTypes[j].c_str(), s->fVariables[j].tag_name.c_str(), s->fVariables[j].tag_type.c_str(), timestamp, true, have_transaction);
5099 0 : if (status != HS_SUCCESS)
5100 0 : return status;
5101 :
5102 0 : if (count == 0) {
5103 0 : s->fOffsets[j] = offset;
5104 0 : offset += rpc_tid_size(tagtype);
5105 : }
5106 0 : count++;
5107 0 : found_column = true;
5108 0 : if (count > 1) {
5109 0 : cm_msg(MERROR, "SqlHistory::update_schema", "Duplicate SQL column \'%s\' type \'%s\' in table \"%s\" for history event \"%s\" tag \"%s\"", s->fColumnNames[j].c_str(), s->fColumnTypes[j].c_str(), s->fTableName.c_str(), s->fEventName.c_str(), tagname.c_str());
5110 0 : cm_msg_flush_buffer();
5111 : }
5112 : }
5113 : }
5114 : }
5115 : }
5116 :
5117 : // create column
5118 0 : if (!found_column && write_enable) {
5119 0 : std::string col_name = maybe_colname;
5120 0 : const char* col_type = s->fSql->ColumnType(tagtype);
5121 :
5122 0 : bool dupe = false;
5123 0 : for (size_t kk=0; kk<s->fColumnNames.size(); kk++)
5124 0 : if (s->fColumnNames[kk] == col_name) {
5125 0 : dupe = true;
5126 0 : break;
5127 : }
5128 :
5129 0 : time_t now = time(NULL);
5130 :
5131 0 : bool retry = false;
5132 0 : for (int t=0; t<20; t++) {
5133 :
5134 : // if duplicate column name, change it, try again
5135 0 : if (dupe || retry) {
5136 0 : col_name = maybe_colname;
5137 0 : col_name += "_";
5138 0 : col_name += TimeToString(now);
5139 0 : if (t > 0) {
5140 : char s[256];
5141 0 : sprintf(s, "_%d", t);
5142 0 : col_name += s;
5143 : }
5144 : }
5145 :
5146 0 : if (fDebug)
5147 0 : printf("SqlHistory::update_schema: table [%s], add column [%s] type [%s] for tag [%s]\n", s->fTableName.c_str(), col_name.c_str(), col_type, tagname.c_str());
5148 :
5149 0 : status = CreateSqlColumn(fSql, s->fTableName.c_str(), col_name.c_str(), col_type, have_transaction, fDebug);
5150 :
5151 0 : if (status == DB_KEY_EXIST) {
5152 0 : if (fDebug)
5153 0 : printf("SqlHistory::update_schema: table [%s], add column [%s] type [%s] for tag [%s] failed: duplicate column name\n", s->fTableName.c_str(), col_name.c_str(), col_type, tagname.c_str());
5154 0 : retry = true;
5155 0 : continue;
5156 : }
5157 :
5158 0 : if (status != HS_SUCCESS)
5159 0 : return status;
5160 :
5161 0 : break;
5162 : }
5163 :
5164 0 : if (status != HS_SUCCESS)
5165 0 : return status;
5166 :
5167 0 : status = update_column(s->fEventName.c_str(), s->fTableName.c_str(), col_name.c_str(), col_type, tagname.c_str(), rpc_tid_name(tagtype), timestamp, true, have_transaction);
5168 0 : if (status != HS_SUCCESS)
5169 0 : return status;
5170 0 : }
5171 : }
5172 :
5173 0 : if (count > 1) {
5174 : // schema has duplicate tags
5175 0 : schema_ok = false;
5176 0 : cm_msg(MERROR, "SqlHistory::update_schema", "Duplicate tags or SQL columns for history event \"%s\" tag \"%s\"", s->fEventName.c_str(), tagname.c_str());
5177 0 : cm_msg_flush_buffer();
5178 : }
5179 0 : }
5180 : }
5181 :
5182 : // mark as unused all columns not listed in tags
5183 :
5184 0 : for (size_t k=0; k<s->fColumnNames.size(); k++)
5185 0 : if (s->fVariables[k].name.length() > 0) {
5186 0 : bool found = false;
5187 :
5188 0 : for (int i=0; i<ntags; i++) {
5189 0 : for (unsigned int j=0; j<tags[i].n_data; j++) {
5190 0 : std::string tagname = tags[i].name;
5191 :
5192 0 : if (tags[i].n_data > 1) {
5193 : char s[256];
5194 0 : sprintf(s, "[%d]", j);
5195 0 : tagname += s;
5196 : }
5197 :
5198 0 : if (s->fVariables[k].name == tagname) {
5199 0 : found = true;
5200 0 : break;
5201 : }
5202 0 : }
5203 :
5204 0 : if (found)
5205 0 : break;
5206 : }
5207 :
5208 0 : if (!found) {
5209 : // column not found in tags list
5210 0 : schema_ok = false;
5211 0 : if (fDebug)
5212 0 : printf("Event [%s] Column [%s] tag [%s] not listed in tags list!\n", s->fEventName.c_str(), s->fColumnNames[k].c_str(), s->fVariables[k].name.c_str());
5213 0 : if (write_enable) {
5214 0 : cm_msg(MINFO, "SqlHistory::update_schema", "Deactivating SQL column \'%s\' type \'%s\' in table \"%s\" for history event \"%s\" not used for any tags", s->fColumnNames[k].c_str(), s->fColumnTypes[k].c_str(), s->fTableName.c_str(), s->fEventName.c_str());
5215 0 : cm_msg_flush_buffer();
5216 :
5217 0 : status = update_column(s->fEventName.c_str(), s->fTableName.c_str(), s->fColumnNames[k].c_str(), s->fColumnTypes[k].c_str(), s->fVariables[k].tag_name.c_str(), s->fVariables[k].tag_type.c_str(), timestamp, false, have_transaction);
5218 0 : if (status != HS_SUCCESS)
5219 0 : return status;
5220 : }
5221 : }
5222 : }
5223 :
5224 0 : if (!write_enable)
5225 0 : if (!schema_ok) {
5226 0 : if (fDebug)
5227 0 : printf("Return error!\n");
5228 0 : return HS_FILE_ERROR;
5229 : }
5230 :
5231 0 : return HS_SUCCESS;
5232 : }
5233 :
5234 : ////////////////////////////////////////////////////////
5235 : // SQLITE functions //
5236 : ////////////////////////////////////////////////////////
5237 :
5238 0 : static int ReadSqliteTableNames(SqlBase* sql, HsSchemaVector *sv, const char* table_name, int debug)
5239 : {
5240 0 : if (debug)
5241 0 : printf("ReadSqliteTableNames: table [%s]\n", table_name);
5242 :
5243 : int status;
5244 0 : std::string cmd;
5245 :
5246 : // FIXME: quotes
5247 0 : cmd = "SELECT event_name, _i_time FROM \'_event_name_";
5248 0 : cmd += table_name;
5249 0 : cmd += "\' WHERE table_name='";
5250 0 : cmd += table_name;
5251 0 : cmd += "';";
5252 :
5253 0 : status = sql->Prepare(table_name, cmd.c_str());
5254 :
5255 0 : if (status != DB_SUCCESS)
5256 0 : return status;
5257 :
5258 : while (1) {
5259 0 : status = sql->Step();
5260 :
5261 0 : if (status != DB_SUCCESS)
5262 0 : break;
5263 :
5264 0 : std::string xevent_name = sql->GetText(0);
5265 0 : time_t xevent_time = sql->GetTime(1);
5266 :
5267 : //printf("read event name [%s] time %s\n", xevent_name.c_str(), TimeToString(xevent_time).c_str());
5268 :
5269 0 : HsSqlSchema* s = new HsSqlSchema;
5270 0 : s->fSql = sql;
5271 0 : s->fEventName = xevent_name;
5272 0 : s->fTimeFrom = xevent_time;
5273 0 : s->fTimeTo = 0;
5274 0 : s->fTableName = table_name;
5275 0 : sv->add(s);
5276 0 : }
5277 :
5278 0 : status = sql->Finalize();
5279 :
5280 0 : return HS_SUCCESS;
5281 0 : }
5282 :
5283 0 : static int ReadSqliteTableSchema(SqlBase* sql, HsSchemaVector *sv, const char* table_name, int debug)
5284 : {
5285 0 : if (debug)
5286 0 : printf("ReadSqliteTableSchema: table [%s]\n", table_name);
5287 :
5288 : if (1) {
5289 : // seed schema with table names
5290 0 : HsSqlSchema* s = new HsSqlSchema;
5291 0 : s->fSql = sql;
5292 0 : s->fEventName = table_name;
5293 0 : s->fTimeFrom = 0;
5294 0 : s->fTimeTo = 0;
5295 0 : s->fTableName = table_name;
5296 0 : sv->add(s);
5297 : }
5298 :
5299 0 : return ReadSqliteTableNames(sql, sv, table_name, debug);
5300 : }
5301 :
5302 : ////////////////////////////////////////////////////////
5303 : // SQLITE history classes //
5304 : ////////////////////////////////////////////////////////
5305 :
5306 : class SqliteHistory: public SqlHistoryBase
5307 : {
5308 : public:
5309 0 : SqliteHistory() { // ctor
5310 : #ifdef HAVE_SQLITE
5311 0 : fSql = new Sqlite();
5312 : #endif
5313 0 : }
5314 :
5315 : int read_table_and_event_names(HsSchemaVector *sv);
5316 : int read_column_names(HsSchemaVector *sv, const char* table_name, const char* event_name);
5317 : int create_table(HsSchemaVector* sv, const char* event_name, time_t timestamp);
5318 : int update_column(const char* event_name, const char* table_name, const char* column_name, const char* column_type, const char* tag_name, const char* tag_type, const time_t timestamp, bool active, bool* have_transaction);
5319 : };
5320 :
5321 0 : int SqliteHistory::read_table_and_event_names(HsSchemaVector *sv)
5322 : {
5323 : int status;
5324 :
5325 0 : if (fDebug)
5326 0 : printf("SqliteHistory::read_table_and_event_names!\n");
5327 :
5328 : // loop over all tables
5329 :
5330 0 : std::vector<std::string> tables;
5331 0 : status = fSql->ListTables(&tables);
5332 0 : if (status != DB_SUCCESS)
5333 0 : return status;
5334 :
5335 0 : for (size_t i=0; i<tables.size(); i++) {
5336 0 : const char* table_name = tables[i].c_str();
5337 :
5338 : const char* s;
5339 0 : s = strstr(table_name, "_event_name_");
5340 0 : if (s == table_name)
5341 0 : continue;
5342 0 : s = strstr(table_name, "_column_names_");
5343 0 : if (s == table_name)
5344 0 : continue;
5345 :
5346 0 : status = ReadSqliteTableSchema(fSql, sv, table_name, fDebug);
5347 : }
5348 :
5349 0 : return HS_SUCCESS;
5350 0 : }
5351 :
5352 0 : int SqliteHistory::read_column_names(HsSchemaVector *sv, const char* table_name, const char* event_name)
5353 : {
5354 0 : if (fDebug)
5355 0 : printf("SqliteHistory::read_column_names: table [%s], event [%s]\n", table_name, event_name);
5356 :
5357 : // for all schema for table_name, prepopulate is with column names
5358 :
5359 0 : std::vector<std::string> columns;
5360 0 : fSql->ListColumns(table_name, &columns);
5361 :
5362 : // first, populate column names
5363 :
5364 0 : for (size_t i=0; i<sv->size(); i++) {
5365 0 : HsSqlSchema* s = (HsSqlSchema*)(*sv)[i];
5366 :
5367 0 : if (s->fTableName != table_name)
5368 0 : continue;
5369 :
5370 : // schema should be empty at this point
5371 : //assert(s->fVariables.size() == 0);
5372 :
5373 0 : for (size_t j=0; j<columns.size(); j+=2) {
5374 0 : const char* cn = columns[j+0].c_str();
5375 0 : const char* ct = columns[j+1].c_str();
5376 :
5377 0 : if (strcmp(cn, "_t_time") == 0)
5378 0 : continue;
5379 0 : if (strcmp(cn, "_i_time") == 0)
5380 0 : continue;
5381 :
5382 0 : bool found = false;
5383 :
5384 0 : for (size_t k=0; k<s->fColumnNames.size(); k++) {
5385 0 : if (s->fColumnNames[k] == cn) {
5386 0 : found = true;
5387 0 : break;
5388 : }
5389 : }
5390 :
5391 : //printf("column [%s] sql type [%s]\n", cn.c_str(), ct);
5392 :
5393 0 : if (!found) {
5394 0 : HsSchemaEntry se;
5395 0 : se.name = cn;
5396 0 : se.type = 0;
5397 0 : se.n_data = 1;
5398 0 : se.n_bytes = 0;
5399 0 : s->fVariables.push_back(se);
5400 0 : s->fColumnNames.push_back(cn);
5401 0 : s->fColumnTypes.push_back(ct);
5402 0 : s->fColumnInactive.push_back(false);
5403 0 : s->fOffsets.push_back(-1);
5404 0 : }
5405 : }
5406 : }
5407 :
5408 : // then read column name information
5409 :
5410 0 : std::string tn;
5411 0 : tn += "_column_names_";
5412 0 : tn += table_name;
5413 :
5414 0 : std::string cmd;
5415 0 : cmd = "SELECT column_name, tag_name, tag_type, _i_time FROM ";
5416 0 : cmd += fSql->QuoteId(tn.c_str());
5417 0 : cmd += " WHERE table_name=";
5418 0 : cmd += fSql->QuoteString(table_name);
5419 0 : cmd += " ORDER BY _i_time ASC;";
5420 :
5421 0 : int status = fSql->Prepare(table_name, cmd.c_str());
5422 :
5423 0 : if (status != DB_SUCCESS) {
5424 0 : return status;
5425 : }
5426 :
5427 : while (1) {
5428 0 : status = fSql->Step();
5429 :
5430 0 : if (status != DB_SUCCESS)
5431 0 : break;
5432 :
5433 : // NOTE: SQL "SELECT ORDER BY _i_time ASC" returns data sorted by time
5434 : // in this code we use the data from the last data row
5435 : // so if multiple rows are present, the latest one is used
5436 :
5437 0 : std::string col_name = fSql->GetText(0);
5438 0 : std::string tag_name = fSql->GetText(1);
5439 0 : std::string tag_type = fSql->GetText(2);
5440 0 : time_t schema_time = fSql->GetTime(3);
5441 :
5442 : //printf("read table [%s] column [%s] tag name [%s] time %s\n", table_name, col_name.c_str(), tag_name.c_str(), TimeToString(xxx_time).c_str());
5443 :
5444 : // make sure a schema exists at this time point
5445 0 : NewSqlSchema(sv, table_name, schema_time);
5446 :
5447 : // add this information to all schema
5448 :
5449 0 : for (size_t i=0; i<sv->size(); i++) {
5450 0 : HsSqlSchema* s = (HsSqlSchema*)(*sv)[i];
5451 0 : if (s->fTableName != table_name)
5452 0 : continue;
5453 0 : if (s->fTimeFrom < schema_time)
5454 0 : continue;
5455 :
5456 : //printf("add column to schema %d\n", s->fTimeFrom);
5457 :
5458 0 : for (size_t j=0; j<s->fColumnNames.size(); j++) {
5459 0 : if (col_name != s->fColumnNames[j])
5460 0 : continue;
5461 0 : s->fVariables[j].name = tag_name;
5462 0 : s->fVariables[j].type = rpc_name_tid(tag_type.c_str());
5463 0 : s->fVariables[j].n_data = 1;
5464 0 : s->fVariables[j].n_bytes = rpc_tid_size(s->fVariables[j].type);
5465 : }
5466 : }
5467 0 : }
5468 :
5469 0 : status = fSql->Finalize();
5470 :
5471 0 : return HS_SUCCESS;
5472 0 : }
5473 :
5474 0 : int SqliteHistory::create_table(HsSchemaVector* sv, const char* event_name, time_t timestamp)
5475 : {
5476 0 : if (fDebug)
5477 0 : printf("SqliteHistory::create_table: event [%s], timestamp %s\n", event_name, TimeToString(timestamp).c_str());
5478 :
5479 : int status;
5480 0 : bool have_transaction = false;
5481 0 : std::string table_name = MidasNameToSqlName(event_name);
5482 :
5483 : // FIXME: what about duplicate table names?
5484 0 : status = CreateSqlTable(fSql, table_name.c_str(), &have_transaction);
5485 :
5486 : //if (status == DB_KEY_EXIST) {
5487 : // return ReadSqliteTableSchema(fSql, sv, table_name.c_str(), fDebug);
5488 : //}
5489 :
5490 0 : if (status != HS_SUCCESS) {
5491 : // FIXME: ???
5492 : // FIXME: at least close or revert the transaction
5493 0 : return status;
5494 : }
5495 :
5496 0 : std::string cmd;
5497 :
5498 0 : std::string en;
5499 0 : en += "_event_name_";
5500 0 : en += table_name;
5501 :
5502 0 : cmd = "CREATE TABLE ";
5503 0 : cmd += fSql->QuoteId(en.c_str());
5504 0 : cmd += " (table_name TEXT NOT NULL, event_name TEXT NOT NULL, _i_time INTEGER NOT NULL);";
5505 :
5506 0 : status = fSql->Exec(table_name.c_str(), cmd.c_str());
5507 :
5508 0 : cmd = "INSERT INTO ";
5509 0 : cmd += fSql->QuoteId(en.c_str());
5510 0 : cmd += " (table_name, event_name, _i_time) VALUES (";
5511 0 : cmd += fSql->QuoteString(table_name.c_str());
5512 0 : cmd += ", ";
5513 0 : cmd += fSql->QuoteString(event_name);
5514 0 : cmd += ", ";
5515 0 : cmd += fSql->QuoteString(TimeToString(timestamp).c_str());
5516 0 : cmd += ");";
5517 :
5518 0 : status = fSql->Exec(table_name.c_str(), cmd.c_str());
5519 :
5520 0 : std::string cn;
5521 0 : cn += "_column_names_";
5522 0 : cn += table_name;
5523 :
5524 0 : cmd = "CREATE TABLE ";
5525 0 : cmd += fSql->QuoteId(cn.c_str());
5526 0 : cmd += " (table_name TEXT NOT NULL, column_name TEXT NOT NULL, tag_name TEXT NOT NULL, tag_type TEXT NOT NULL, column_type TEXT NOT NULL, _i_time INTEGER NOT NULL);";
5527 :
5528 0 : status = fSql->Exec(table_name.c_str(), cmd.c_str());
5529 :
5530 0 : status = fSql->CommitTransaction(table_name.c_str());
5531 0 : if (status != DB_SUCCESS) {
5532 0 : return HS_FILE_ERROR;
5533 : }
5534 :
5535 0 : return ReadSqliteTableSchema(fSql, sv, table_name.c_str(), fDebug);
5536 0 : }
5537 :
5538 0 : int SqliteHistory::update_column(const char* event_name, const char* table_name, const char* column_name, const char* column_type, const char* tag_name, const char* tag_type, const time_t timestamp, bool active, bool* have_transaction)
5539 : {
5540 0 : if (fDebug)
5541 0 : printf("SqliteHistory::update_column: event [%s], table [%s], column [%s], new name [%s], timestamp %s\n", event_name, table_name, column_name, tag_name, TimeToString(timestamp).c_str());
5542 :
5543 0 : int status = StartSqlTransaction(fSql, table_name, have_transaction);
5544 0 : if (status != HS_SUCCESS)
5545 0 : return status;
5546 :
5547 : // FIXME: quotes
5548 0 : std::string cmd;
5549 0 : cmd = "INSERT INTO \'_column_names_";
5550 0 : cmd += table_name;
5551 0 : cmd += "\' (table_name, column_name, tag_name, tag_type, column_type, _i_time) VALUES (\'";
5552 0 : cmd += table_name;
5553 0 : cmd += "\', \'";
5554 0 : cmd += column_name;
5555 0 : cmd += "\', \'";
5556 0 : cmd += tag_name;
5557 0 : cmd += "\', \'";
5558 0 : cmd += tag_type;
5559 0 : cmd += "\', \'";
5560 0 : cmd += column_type;
5561 0 : cmd += "\', \'";
5562 0 : cmd += TimeToString(timestamp);
5563 0 : cmd += "\');";
5564 0 : status = fSql->Exec(table_name, cmd.c_str());
5565 :
5566 0 : return status;
5567 0 : }
5568 :
5569 : ////////////////////////////////////////////////////////
5570 : // Mysql history classes //
5571 : ////////////////////////////////////////////////////////
5572 :
5573 : class MysqlHistory: public SqlHistoryBase
5574 : {
5575 : public:
5576 0 : MysqlHistory() { // ctor
5577 : #ifdef HAVE_MYSQL
5578 0 : fSql = new Mysql();
5579 : #endif
5580 0 : }
5581 :
5582 : int read_table_and_event_names(HsSchemaVector *sv);
5583 : int read_column_names(HsSchemaVector *sv, const char* table_name, const char* event_name);
5584 : int create_table(HsSchemaVector* sv, const char* event_name, time_t timestamp);
5585 : int update_column(const char* event_name, const char* table_name, const char* column_name, const char* column_type, const char* tag_name, const char* tag_type, const time_t timestamp, bool active, bool* have_transaction);
5586 : };
5587 :
5588 0 : static int ReadMysqlTableNames(SqlBase* sql, HsSchemaVector *sv, const char* table_name, int debug, const char* must_have_event_name, const char* must_have_table_name)
5589 : {
5590 0 : if (debug)
5591 0 : printf("ReadMysqlTableNames: table [%s], must have event [%s] table [%s]\n", table_name, must_have_event_name, must_have_table_name);
5592 :
5593 : int status;
5594 0 : std::string cmd;
5595 :
5596 0 : if (table_name) {
5597 0 : cmd = "SELECT event_name, table_name, itimestamp FROM _history_index WHERE table_name='";
5598 0 : cmd += table_name;
5599 0 : cmd += "';";
5600 : } else {
5601 0 : cmd = "SELECT event_name, table_name, itimestamp FROM _history_index WHERE table_name!='';";
5602 0 : table_name = "_history_index";
5603 : }
5604 :
5605 0 : status = sql->Prepare(table_name, cmd.c_str());
5606 :
5607 0 : if (status != DB_SUCCESS)
5608 0 : return status;
5609 :
5610 0 : bool found_must_have_table = false;
5611 0 : int count = 0;
5612 :
5613 : while (1) {
5614 0 : status = sql->Step();
5615 :
5616 0 : if (status != DB_SUCCESS)
5617 0 : break;
5618 :
5619 0 : const char* xevent_name = sql->GetText(0);
5620 0 : const char* xtable_name = sql->GetText(1);
5621 0 : time_t xevent_time = sql->GetTime(2);
5622 :
5623 0 : if (debug == 999) {
5624 0 : printf("entry %d event name [%s] table name [%s] time %s\n", count, xevent_name, xtable_name, TimeToString(xevent_time).c_str());
5625 : }
5626 :
5627 0 : if (must_have_table_name && (strcmp(xtable_name, must_have_table_name) == 0)) {
5628 0 : assert(must_have_event_name != NULL);
5629 0 : if (event_name_cmp(xevent_name, must_have_event_name) == 0) {
5630 0 : found_must_have_table = true;
5631 : //printf("Found table [%s]: event name [%s] table name [%s] time %s\n", must_have_table_name, xevent_name, xtable_name, TimeToString(xevent_time).c_str());
5632 : } else {
5633 : //printf("Found correct table [%s] with wrong event name [%s] expected [%s] time %s\n", must_have_table_name, xevent_name, must_have_event_name, TimeToString(xevent_time).c_str());
5634 : }
5635 : }
5636 :
5637 0 : HsSqlSchema* s = new HsSqlSchema;
5638 0 : s->fSql = sql;
5639 0 : s->fEventName = xevent_name;
5640 0 : s->fTimeFrom = xevent_time;
5641 0 : s->fTimeTo = 0;
5642 0 : s->fTableName = xtable_name;
5643 0 : sv->add(s);
5644 0 : count++;
5645 0 : }
5646 :
5647 0 : status = sql->Finalize();
5648 :
5649 0 : if (must_have_table_name && !found_must_have_table) {
5650 0 : cm_msg(MERROR, "ReadMysqlTableNames", "Error: Table [%s] for event [%s] missing from the history index\n", must_have_table_name, must_have_event_name);
5651 0 : if (debug == 999)
5652 0 : return HS_FILE_ERROR;
5653 : // NB: recursion is broken by setting debug to 999.
5654 0 : ReadMysqlTableNames(sql, sv, table_name, 999, must_have_event_name, must_have_table_name);
5655 0 : cm_msg(MERROR, "ReadMysqlTableNames", "Error: Cannot continue, nothing will work after this error\n");
5656 0 : cm_msg_flush_buffer();
5657 0 : abort();
5658 : return HS_FILE_ERROR;
5659 : }
5660 :
5661 : if (0) {
5662 : // print accumulated schema
5663 : printf("ReadMysqlTableNames: table_name [%s] event_name [%s] table_name [%s]\n", table_name, must_have_event_name, must_have_table_name);
5664 : sv->print(false);
5665 : }
5666 :
5667 0 : return HS_SUCCESS;
5668 0 : }
5669 :
5670 0 : int MysqlHistory::read_column_names(HsSchemaVector *sv, const char* table_name, const char* event_name)
5671 : {
5672 0 : if (fDebug)
5673 0 : printf("MysqlHistory::read_column_names: table [%s], event [%s]\n", table_name, event_name);
5674 :
5675 : // for all schema for table_name, prepopulate is with column names
5676 :
5677 0 : std::vector<std::string> columns;
5678 0 : fSql->ListColumns(table_name, &columns);
5679 :
5680 : // first, populate column names
5681 :
5682 0 : for (size_t i=0; i<sv->size(); i++) {
5683 0 : HsSqlSchema* s = (HsSqlSchema*)(*sv)[i];
5684 :
5685 0 : if (s->fTableName != table_name)
5686 0 : continue;
5687 :
5688 : // schema should be empty at this point
5689 : //assert(s->fVariables.size() == 0);
5690 :
5691 0 : for (size_t j=0; j<columns.size(); j+=2) {
5692 0 : const char* cn = columns[j+0].c_str();
5693 0 : const char* ct = columns[j+1].c_str();
5694 :
5695 0 : if (strcmp(cn, "_t_time") == 0)
5696 0 : continue;
5697 0 : if (strcmp(cn, "_i_time") == 0)
5698 0 : continue;
5699 :
5700 0 : bool found = false;
5701 :
5702 0 : for (size_t k=0; k<s->fColumnNames.size(); k++) {
5703 0 : if (s->fColumnNames[k] == cn) {
5704 0 : found = true;
5705 0 : break;
5706 : }
5707 : }
5708 :
5709 : //printf("column [%s] sql type [%s]\n", cn.c_str(), ct);
5710 :
5711 0 : if (!found) {
5712 0 : HsSchemaEntry se;
5713 0 : se.tag_name = cn;
5714 0 : se.tag_type = "";
5715 0 : se.name = cn;
5716 0 : se.type = 0;
5717 0 : se.n_data = 1;
5718 0 : se.n_bytes = 0;
5719 0 : s->fVariables.push_back(se);
5720 0 : s->fColumnNames.push_back(cn);
5721 0 : s->fColumnTypes.push_back(ct);
5722 0 : s->fColumnInactive.push_back(false);
5723 0 : s->fOffsets.push_back(-1);
5724 0 : }
5725 : }
5726 : }
5727 :
5728 : // then read column name information
5729 :
5730 0 : std::string cmd;
5731 0 : cmd = "SELECT column_name, column_type, tag_name, tag_type, itimestamp, active FROM _history_index WHERE event_name=";
5732 0 : cmd += fSql->QuoteString(event_name);
5733 0 : cmd += ";";
5734 :
5735 0 : int status = fSql->Prepare(table_name, cmd.c_str());
5736 :
5737 0 : if (status != DB_SUCCESS) {
5738 0 : return status;
5739 : }
5740 :
5741 : while (1) {
5742 0 : status = fSql->Step();
5743 :
5744 0 : if (status != DB_SUCCESS)
5745 0 : break;
5746 :
5747 0 : const char* col_name = fSql->GetText(0);
5748 0 : const char* col_type = fSql->GetText(1);
5749 0 : const char* tag_name = fSql->GetText(2);
5750 0 : const char* tag_type = fSql->GetText(3);
5751 0 : time_t schema_time = fSql->GetTime(4);
5752 0 : const char* active = fSql->GetText(5);
5753 0 : int iactive = atoi(active);
5754 :
5755 : //printf("read table [%s] column [%s] type [%s] tag name [%s] type [%s] time %s active [%s] %d\n", table_name, col_name, col_type, tag_name, tag_type, TimeToString(schema_time).c_str(), active, iactive);
5756 :
5757 0 : if (!col_name)
5758 0 : continue;
5759 0 : if (!tag_name)
5760 0 : continue;
5761 0 : if (strlen(col_name) < 1)
5762 0 : continue;
5763 :
5764 : // make sure a schema exists at this time point
5765 0 : NewSqlSchema(sv, table_name, schema_time);
5766 :
5767 : // add this information to all schema
5768 :
5769 0 : for (size_t i=0; i<sv->size(); i++) {
5770 0 : HsSqlSchema* s = (HsSqlSchema*)(*sv)[i];
5771 0 : if (s->fTableName != table_name)
5772 0 : continue;
5773 0 : if (s->fTimeFrom < schema_time)
5774 0 : continue;
5775 :
5776 0 : int tid = rpc_name_tid(tag_type);
5777 0 : int tid_size = rpc_tid_size(tid);
5778 :
5779 0 : for (size_t j=0; j<s->fColumnNames.size(); j++) {
5780 0 : if (col_name != s->fColumnNames[j])
5781 0 : continue;
5782 :
5783 0 : s->fVariables[j].tag_name = tag_name;
5784 0 : s->fVariables[j].tag_type = tag_type;
5785 0 : if (!iactive) {
5786 0 : s->fVariables[j].name = "";
5787 0 : s->fColumnInactive[j] = true;
5788 : } else {
5789 0 : s->fVariables[j].name = tag_name;
5790 0 : s->fColumnInactive[j] = false;
5791 : }
5792 0 : s->fVariables[j].type = tid;
5793 0 : s->fVariables[j].n_data = 1;
5794 0 : s->fVariables[j].n_bytes = tid_size;
5795 :
5796 : // doctor column names in case MySQL returns different type
5797 : // from the type used to create the column, but the types
5798 : // are actually the same. K.O.
5799 0 : DoctorSqlColumnType(&s->fColumnTypes[j], col_type);
5800 : }
5801 : }
5802 0 : }
5803 :
5804 0 : status = fSql->Finalize();
5805 :
5806 0 : return HS_SUCCESS;
5807 0 : }
5808 :
5809 : #if 0
5810 : static int ReadMysqlTableSchema(SqlBase* sql, HsSchemaVector *sv, const char* table_name, int debug)
5811 : {
5812 : if (debug)
5813 : printf("ReadMysqlTableSchema: table [%s]\n", table_name);
5814 :
5815 : if (1) {
5816 : // seed schema with table names
5817 : HsSqlSchema* s = new HsSqlSchema;
5818 : s->fSql = sql;
5819 : s->fEventName = table_name;
5820 : s->fTimeFrom = 0;
5821 : s->fTimeTo = 0;
5822 : s->fTableName = table_name;
5823 : sv->add(s);
5824 : }
5825 :
5826 : return ReadMysqlTableNames(sql, sv, table_name, debug, NULL, NULL);
5827 : }
5828 : #endif
5829 :
5830 0 : int MysqlHistory::read_table_and_event_names(HsSchemaVector *sv)
5831 : {
5832 : int status;
5833 :
5834 0 : if (fDebug)
5835 0 : printf("MysqlHistory::read_table_and_event_names!\n");
5836 :
5837 : // loop over all tables
5838 :
5839 0 : std::vector<std::string> tables;
5840 0 : status = fSql->ListTables(&tables);
5841 0 : if (status != DB_SUCCESS)
5842 0 : return status;
5843 :
5844 0 : for (size_t i=0; i<tables.size(); i++) {
5845 0 : const char* table_name = tables[i].c_str();
5846 :
5847 : const char* s;
5848 0 : s = strstr(table_name, "_history_index");
5849 0 : if (s == table_name)
5850 0 : continue;
5851 :
5852 : if (1) {
5853 : // seed schema with table names
5854 0 : HsSqlSchema* s = new HsSqlSchema;
5855 0 : s->fSql = fSql;
5856 0 : s->fEventName = table_name;
5857 0 : s->fTimeFrom = 0;
5858 0 : s->fTimeTo = 0;
5859 0 : s->fTableName = table_name;
5860 0 : sv->add(s);
5861 : }
5862 : }
5863 :
5864 : if (0) {
5865 : // print accumulated schema
5866 : printf("read_table_and_event_names:\n");
5867 : sv->print(false);
5868 : }
5869 :
5870 0 : status = ReadMysqlTableNames(fSql, sv, NULL, fDebug, NULL, NULL);
5871 :
5872 0 : return HS_SUCCESS;
5873 0 : }
5874 :
5875 0 : int MysqlHistory::create_table(HsSchemaVector* sv, const char* event_name, time_t timestamp)
5876 : {
5877 0 : if (fDebug)
5878 0 : printf("MysqlHistory::create_table: event [%s], timestamp %s\n", event_name, TimeToString(timestamp).c_str());
5879 :
5880 : int status;
5881 0 : std::string table_name = MidasNameToSqlName(event_name);
5882 :
5883 : // MySQL table name length limit is 64 bytes
5884 0 : if (table_name.length() > 40) {
5885 0 : table_name.resize(40);
5886 0 : table_name += "_T";
5887 : }
5888 :
5889 0 : time_t now = time(NULL);
5890 :
5891 0 : int max_attempts = 10;
5892 0 : for (int i=0; i<max_attempts; i++) {
5893 0 : status = fSql->OpenTransaction(table_name.c_str());
5894 0 : if (status != DB_SUCCESS) {
5895 0 : return HS_FILE_ERROR;
5896 : }
5897 :
5898 0 : bool have_transaction = true;
5899 :
5900 0 : std::string xtable_name = table_name;
5901 :
5902 0 : if (i>0) {
5903 0 : xtable_name += "_";
5904 0 : xtable_name += TimeToString(now);
5905 0 : if (i>1) {
5906 0 : xtable_name += "_";
5907 : char buf[256];
5908 0 : sprintf(buf, "%d", i);
5909 0 : xtable_name += buf;
5910 : }
5911 : }
5912 :
5913 0 : status = CreateSqlTable(fSql, xtable_name.c_str(), &have_transaction);
5914 :
5915 : //printf("event [%s] create table [%s] status %d\n", event_name, xtable_name.c_str(), status);
5916 :
5917 0 : if (status == DB_KEY_EXIST) {
5918 : // already exists, try with different name!
5919 0 : fSql->RollbackTransaction(table_name.c_str());
5920 0 : continue;
5921 : }
5922 :
5923 0 : if (status != HS_SUCCESS) {
5924 : // MYSQL cannot roll back "create table", if we cannot create SQL tables, nothing will work. Give up now.
5925 0 : cm_msg(MERROR, "MysqlHistory::create_table", "Could not create table [%s] for event [%s], timestamp %s, please fix the SQL database configuration and try again", table_name.c_str(), event_name, TimeToString(timestamp).c_str());
5926 0 : abort();
5927 :
5928 : // fatal error, give up!
5929 : fSql->RollbackTransaction(table_name.c_str());
5930 : break;
5931 : }
5932 :
5933 0 : for (int j=0; j<2; j++) {
5934 0 : std::string cmd;
5935 0 : cmd += "INSERT INTO _history_index (event_name, table_name, itimestamp, active) VALUES (";
5936 0 : cmd += fSql->QuoteString(event_name);
5937 0 : cmd += ", ";
5938 0 : cmd += fSql->QuoteString(xtable_name.c_str());
5939 0 : cmd += ", ";
5940 : char buf[256];
5941 0 : sprintf(buf, "%.0f", (double)timestamp);
5942 0 : cmd += fSql->QuoteString(buf);
5943 0 : cmd += ", ";
5944 0 : cmd += fSql->QuoteString("1");
5945 0 : cmd += ");";
5946 :
5947 0 : int status = fSql->Exec(table_name.c_str(), cmd.c_str());
5948 0 : if (status == DB_SUCCESS)
5949 0 : break;
5950 :
5951 0 : status = CreateSqlTable(fSql, "_history_index", &have_transaction);
5952 0 : status = CreateSqlColumn(fSql, "_history_index", "event_name", "varchar(256) character set binary not null", &have_transaction, fDebug);
5953 0 : status = CreateSqlColumn(fSql, "_history_index", "table_name", "varchar(256)", &have_transaction, fDebug);
5954 0 : status = CreateSqlColumn(fSql, "_history_index", "tag_name", "varchar(256) character set binary", &have_transaction, fDebug);
5955 0 : status = CreateSqlColumn(fSql, "_history_index", "tag_type", "varchar(256)", &have_transaction, fDebug);
5956 0 : status = CreateSqlColumn(fSql, "_history_index", "column_name", "varchar(256)", &have_transaction, fDebug);
5957 0 : status = CreateSqlColumn(fSql, "_history_index", "column_type", "varchar(256)", &have_transaction, fDebug);
5958 0 : status = CreateSqlColumn(fSql, "_history_index", "itimestamp", "integer not null", &have_transaction, fDebug);
5959 0 : status = CreateSqlColumn(fSql, "_history_index", "active", "boolean", &have_transaction, fDebug);
5960 0 : }
5961 :
5962 0 : status = fSql->CommitTransaction(table_name.c_str());
5963 :
5964 0 : if (status != DB_SUCCESS) {
5965 0 : return HS_FILE_ERROR;
5966 : }
5967 :
5968 0 : return ReadMysqlTableNames(fSql, sv, xtable_name.c_str(), fDebug, event_name, xtable_name.c_str());
5969 0 : }
5970 :
5971 0 : cm_msg(MERROR, "MysqlHistory::create_table", "Could not create table [%s] for event [%s], timestamp %s, after %d attempts", table_name.c_str(), event_name, TimeToString(timestamp).c_str(), max_attempts);
5972 :
5973 0 : return HS_FILE_ERROR;
5974 0 : }
5975 :
5976 0 : int MysqlHistory::update_column(const char* event_name, const char* table_name, const char* column_name, const char* column_type, const char* tag_name, const char* tag_type, const time_t timestamp, bool active, bool* have_transaction)
5977 : {
5978 0 : if (fDebug)
5979 0 : printf("MysqlHistory::update_column: event [%s], table [%s], column [%s], type [%s] new name [%s], timestamp %s\n", event_name, table_name, column_name, column_type, tag_name, TimeToString(timestamp).c_str());
5980 :
5981 0 : std::string cmd;
5982 0 : cmd += "INSERT INTO _history_index (event_name, table_name, tag_name, tag_type, column_name, column_type, itimestamp, active) VALUES (";
5983 0 : cmd += fSql->QuoteString(event_name);
5984 0 : cmd += ", ";
5985 0 : cmd += fSql->QuoteString(table_name);
5986 0 : cmd += ", ";
5987 0 : cmd += fSql->QuoteString(tag_name);
5988 0 : cmd += ", ";
5989 0 : cmd += fSql->QuoteString(tag_type);
5990 0 : cmd += ", ";
5991 0 : cmd += fSql->QuoteString(column_name);
5992 0 : cmd += ", ";
5993 0 : cmd += fSql->QuoteString(column_type);
5994 0 : cmd += ", ";
5995 : char buf[256];
5996 0 : sprintf(buf, "%.0f", (double)timestamp);
5997 0 : cmd += fSql->QuoteString(buf);
5998 0 : cmd += ", ";
5999 0 : if (active)
6000 0 : cmd += fSql->QuoteString("1");
6001 : else
6002 0 : cmd += fSql->QuoteString("0");
6003 0 : cmd += ");";
6004 :
6005 0 : int status = fSql->Exec(table_name, cmd.c_str());
6006 0 : if (status != DB_SUCCESS)
6007 0 : return HS_FILE_ERROR;
6008 :
6009 0 : return HS_SUCCESS;
6010 0 : }
6011 :
6012 : ////////////////////////////////////////////////////////
6013 : // PostgreSQL history classes //
6014 : ////////////////////////////////////////////////////////
6015 :
6016 : #ifdef HAVE_PGSQL
6017 :
6018 : class PgsqlHistory: public SqlHistoryBase
6019 : {
6020 : public:
6021 : Pgsql *fPgsql = NULL;
6022 : public:
6023 0 : PgsqlHistory() { // ctor
6024 0 : fPgsql = new Pgsql();
6025 0 : fSql = fPgsql;
6026 0 : }
6027 :
6028 : int read_table_and_event_names(HsSchemaVector *sv);
6029 : int read_column_names(HsSchemaVector *sv, const char* table_name, const char* event_name);
6030 : int create_table(HsSchemaVector* sv, const char* event_name, time_t timestamp);
6031 : int update_column(const char* event_name, const char* table_name, const char* column_name, const char* column_type, const char* tag_name, const char* tag_type, const time_t timestamp, bool active, bool* have_transaction);
6032 : };
6033 :
6034 0 : static int ReadPgsqlTableNames(SqlBase* sql, HsSchemaVector *sv, const char* table_name, int debug, const char* must_have_event_name, const char* must_have_table_name)
6035 : {
6036 0 : if (debug)
6037 0 : printf("ReadPgsqlTableNames: table [%s], must have event [%s] table [%s]\n", table_name, must_have_event_name, must_have_table_name);
6038 :
6039 : int status;
6040 0 : std::string cmd;
6041 :
6042 0 : if (table_name) {
6043 0 : cmd = "SELECT event_name, table_name, itimestamp FROM _history_index WHERE table_name='";
6044 0 : cmd += table_name;
6045 0 : cmd += "';";
6046 : } else {
6047 0 : cmd = "SELECT event_name, table_name, itimestamp FROM _history_index WHERE table_name!='';";
6048 0 : table_name = "_history_index";
6049 : }
6050 :
6051 0 : status = sql->Prepare(table_name, cmd.c_str());
6052 :
6053 0 : if (status != DB_SUCCESS)
6054 0 : return status;
6055 :
6056 0 : bool found_must_have_table = false;
6057 0 : int count = 0;
6058 :
6059 : while (1) {
6060 0 : status = sql->Step();
6061 :
6062 0 : if (status != DB_SUCCESS)
6063 0 : break;
6064 :
6065 0 : const char* xevent_name = sql->GetText(0);
6066 0 : const char* xtable_name = sql->GetText(1);
6067 0 : time_t xevent_time = sql->GetTime(2);
6068 :
6069 0 : if (debug == 999) {
6070 0 : printf("entry %d event name [%s] table name [%s] time %s\n", count, xevent_name, xtable_name, TimeToString(xevent_time).c_str());
6071 : }
6072 :
6073 0 : if (must_have_table_name && (strcmp(xtable_name, must_have_table_name) == 0)) {
6074 0 : assert(must_have_event_name != NULL);
6075 0 : if (event_name_cmp(xevent_name, must_have_event_name) == 0) {
6076 0 : found_must_have_table = true;
6077 : //printf("Found table [%s]: event name [%s] table name [%s] time %s\n", must_have_table_name, xevent_name, xtable_name, TimeToString(xevent_time).c_str());
6078 : } else {
6079 : //printf("Found correct table [%s] with wrong event name [%s] expected [%s] time %s\n", must_have_table_name, xevent_name, must_have_event_name, TimeToString(xevent_time).c_str());
6080 : }
6081 : }
6082 :
6083 0 : HsSqlSchema* s = new HsSqlSchema;
6084 0 : s->fSql = sql;
6085 0 : s->fEventName = xevent_name;
6086 0 : s->fTimeFrom = xevent_time;
6087 0 : s->fTimeTo = 0;
6088 0 : s->fTableName = xtable_name;
6089 0 : sv->add(s);
6090 0 : count++;
6091 0 : }
6092 :
6093 0 : status = sql->Finalize();
6094 :
6095 0 : if (must_have_table_name && !found_must_have_table) {
6096 0 : cm_msg(MERROR, "ReadPgsqlTableNames", "Error: Table [%s] for event [%s] missing from the history index\n", must_have_table_name, must_have_event_name);
6097 0 : if (debug == 999)
6098 0 : return HS_FILE_ERROR;
6099 : // NB: recursion is broken by setting debug to 999.
6100 0 : ReadPgsqlTableNames(sql, sv, table_name, 999, must_have_event_name, must_have_table_name);
6101 0 : cm_msg(MERROR, "ReadPgsqlTableNames", "Error: Cannot continue, nothing will work after this error\n");
6102 0 : cm_msg_flush_buffer();
6103 0 : abort();
6104 : return HS_FILE_ERROR;
6105 : }
6106 :
6107 : if (0) {
6108 : // print accumulated schema
6109 : printf("ReadPgsqlTableNames: table_name [%s] event_name [%s] table_name [%s]\n", table_name, must_have_event_name, must_have_table_name);
6110 : sv->print(false);
6111 : }
6112 :
6113 0 : return HS_SUCCESS;
6114 0 : }
6115 :
6116 0 : int PgsqlHistory::read_column_names(HsSchemaVector *sv, const char* table_name, const char* event_name)
6117 : {
6118 0 : if (fDebug)
6119 0 : printf("PgsqlHistory::read_column_names: table [%s], event [%s]\n", table_name, event_name);
6120 :
6121 : // for all schema for table_name, prepopulate is with column names
6122 :
6123 0 : std::vector<std::string> columns;
6124 0 : fSql->ListColumns(table_name, &columns);
6125 :
6126 : // first, populate column names
6127 :
6128 0 : for (size_t i=0; i<sv->size(); i++) {
6129 0 : HsSqlSchema* s = (HsSqlSchema*)(*sv)[i];
6130 :
6131 0 : if (s->fTableName != table_name)
6132 0 : continue;
6133 :
6134 : // schema should be empty at this point
6135 : //assert(s->fVariables.size() == 0);
6136 :
6137 0 : for (size_t j=0; j<columns.size(); j+=2) {
6138 0 : const char* cn = columns[j+0].c_str();
6139 0 : const char* ct = columns[j+1].c_str();
6140 :
6141 0 : if (strcmp(cn, "_t_time") == 0)
6142 0 : continue;
6143 0 : if (strcmp(cn, "_i_time") == 0)
6144 0 : continue;
6145 :
6146 0 : bool found = false;
6147 :
6148 0 : for (size_t k=0; k<s->fColumnNames.size(); k++) {
6149 0 : if (s->fColumnNames[k] == cn) {
6150 0 : found = true;
6151 0 : break;
6152 : }
6153 : }
6154 :
6155 0 : if (!found) {
6156 0 : HsSchemaEntry se;
6157 0 : se.tag_name = cn;
6158 0 : se.tag_type = "";
6159 0 : se.name = cn;
6160 0 : se.type = 0;
6161 0 : se.n_data = 1;
6162 0 : se.n_bytes = 0;
6163 0 : s->fVariables.push_back(se);
6164 0 : s->fColumnNames.push_back(cn);
6165 0 : s->fColumnTypes.push_back(ct);
6166 0 : s->fColumnInactive.push_back(false);
6167 0 : s->fOffsets.push_back(-1);
6168 0 : }
6169 : }
6170 : }
6171 :
6172 : // then read column name information
6173 :
6174 0 : std::string cmd;
6175 0 : cmd = "SELECT column_name, column_type, tag_name, tag_type, itimestamp, active FROM _history_index WHERE event_name=";
6176 0 : cmd += fSql->QuoteString(event_name);
6177 0 : cmd += ";";
6178 :
6179 0 : int status = fSql->Prepare(table_name, cmd.c_str());
6180 :
6181 0 : if (status != DB_SUCCESS) {
6182 0 : return status;
6183 : }
6184 :
6185 : while (1) {
6186 0 : status = fSql->Step();
6187 :
6188 0 : if (status != DB_SUCCESS)
6189 0 : break;
6190 :
6191 0 : const char* col_name = fSql->GetText(0);
6192 0 : const char* col_type = fSql->GetText(1);
6193 0 : const char* tag_name = fSql->GetText(2);
6194 0 : const char* tag_type = fSql->GetText(3);
6195 0 : time_t schema_time = fSql->GetTime(4);
6196 0 : const char* active = fSql->GetText(5);
6197 0 : int iactive = atoi(active);
6198 :
6199 : //printf("read table [%s] column [%s] type [%s] tag name [%s] type [%s] time %s active [%s] %d\n", table_name, col_name, col_type, tag_name, tag_type, TimeToString(schema_time).c_str(), active, iactive);
6200 :
6201 0 : if (!col_name)
6202 0 : continue;
6203 0 : if (!tag_name)
6204 0 : continue;
6205 0 : if (strlen(col_name) < 1)
6206 0 : continue;
6207 :
6208 : // make sure a schema exists at this time point
6209 0 : NewSqlSchema(sv, table_name, schema_time);
6210 :
6211 : // add this information to all schema
6212 0 : for (size_t i=0; i<sv->size(); i++) {
6213 0 : HsSqlSchema* s = (HsSqlSchema*)(*sv)[i];
6214 0 : if (s->fTableName != table_name)
6215 0 : continue;
6216 0 : if (s->fTimeFrom < schema_time)
6217 0 : continue;
6218 :
6219 0 : int tid = rpc_name_tid(tag_type);
6220 0 : int tid_size = rpc_tid_size(tid);
6221 :
6222 0 : for (size_t j=0; j<s->fColumnNames.size(); j++) {
6223 0 : if (col_name != s->fColumnNames[j])
6224 0 : continue;
6225 :
6226 0 : s->fVariables[j].tag_name = tag_name;
6227 0 : s->fVariables[j].tag_type = tag_type;
6228 0 : if (!iactive) {
6229 0 : s->fVariables[j].name = "";
6230 0 : s->fColumnInactive[j] = true;
6231 : } else {
6232 0 : s->fVariables[j].name = tag_name;
6233 0 : s->fColumnInactive[j] = false;
6234 : }
6235 0 : s->fVariables[j].type = tid;
6236 0 : s->fVariables[j].n_data = 1;
6237 0 : s->fVariables[j].n_bytes = tid_size;
6238 :
6239 : // doctor column names in case MySQL returns different type
6240 : // from the type used to create the column, but the types
6241 : // are actually the same. K.O.
6242 0 : DoctorPgsqlColumnType(&s->fColumnTypes[j], col_type);
6243 : }
6244 : }
6245 0 : }
6246 :
6247 0 : status = fSql->Finalize();
6248 :
6249 0 : return HS_SUCCESS;
6250 0 : }
6251 :
6252 0 : int PgsqlHistory::read_table_and_event_names(HsSchemaVector *sv)
6253 : {
6254 : int status;
6255 :
6256 0 : if (fDebug)
6257 0 : printf("PgsqlHistory::read_table_and_event_names!\n");
6258 :
6259 : // loop over all tables
6260 :
6261 0 : std::vector<std::string> tables;
6262 0 : status = fSql->ListTables(&tables);
6263 0 : if (status != DB_SUCCESS)
6264 0 : return status;
6265 :
6266 0 : for (size_t i=0; i<tables.size(); i++) {
6267 0 : const char* table_name = tables[i].c_str();
6268 :
6269 : const char* s;
6270 0 : s = strstr(table_name, "_history_index");
6271 0 : if (s == table_name)
6272 0 : continue;
6273 :
6274 : if (1) {
6275 : // seed schema with table names
6276 0 : HsSqlSchema* s = new HsSqlSchema;
6277 0 : s->fSql = fSql;
6278 0 : s->fEventName = table_name;
6279 0 : s->fTimeFrom = 0;
6280 0 : s->fTimeTo = 0;
6281 0 : s->fTableName = table_name;
6282 0 : sv->add(s);
6283 : }
6284 : }
6285 :
6286 : if (0) {
6287 : // print accumulated schema
6288 : printf("read_table_and_event_names:\n");
6289 : sv->print(false);
6290 : }
6291 :
6292 0 : status = ReadPgsqlTableNames(fSql, sv, NULL, fDebug, NULL, NULL);
6293 :
6294 0 : return HS_SUCCESS;
6295 0 : }
6296 :
6297 0 : int PgsqlHistory::create_table(HsSchemaVector* sv, const char* event_name, time_t timestamp)
6298 : {
6299 0 : if (fDebug)
6300 0 : printf("PgsqlHistory::create_table: event [%s], timestamp %s\n", event_name, TimeToString(timestamp).c_str());
6301 :
6302 : int status;
6303 0 : std::string table_name = MidasNameToSqlName(event_name);
6304 :
6305 : // PostgreSQL table name length limit is 64 bytes
6306 0 : if (table_name.length() > 40) {
6307 0 : table_name.resize(40);
6308 0 : table_name += "_T";
6309 : }
6310 :
6311 0 : time_t now = time(NULL);
6312 :
6313 0 : int max_attempts = 10;
6314 0 : for (int i=0; i<max_attempts; i++) {
6315 0 : status = fSql->OpenTransaction(table_name.c_str());
6316 0 : if (status != DB_SUCCESS) {
6317 0 : return HS_FILE_ERROR;
6318 : }
6319 :
6320 0 : bool have_transaction = true;
6321 :
6322 0 : std::string xtable_name = table_name;
6323 :
6324 0 : if (i>0) {
6325 0 : xtable_name += "_";
6326 0 : xtable_name += TimeToString(now);
6327 0 : if (i>1) {
6328 0 : xtable_name += "_";
6329 : char buf[256];
6330 0 : sprintf(buf, "%d", i);
6331 0 : xtable_name += buf;
6332 : }
6333 : }
6334 :
6335 0 : if (fPgsql->fDownsample)
6336 0 : status = CreateSqlHyperTable(fSql, xtable_name.c_str(), &have_transaction);
6337 : else
6338 0 : status = CreateSqlTable(fSql, xtable_name.c_str(), &have_transaction);
6339 :
6340 : //printf("event [%s] create table [%s] status %d\n", event_name, xtable_name.c_str(), status);
6341 :
6342 0 : if (status == DB_KEY_EXIST) {
6343 : // already exists, try with different name!
6344 0 : fSql->RollbackTransaction(table_name.c_str());
6345 0 : continue;
6346 : }
6347 :
6348 0 : if (status != HS_SUCCESS) {
6349 0 : fSql->RollbackTransaction(table_name.c_str());
6350 0 : continue;
6351 : }
6352 :
6353 0 : fSql->Exec(table_name.c_str(), "SAVEPOINT t0");
6354 :
6355 0 : for (int j=0; j<2; j++) {
6356 0 : std::string cmd;
6357 0 : cmd += "INSERT INTO _history_index (event_name, table_name, itimestamp, active) VALUES (";
6358 0 : cmd += fSql->QuoteString(event_name);
6359 0 : cmd += ", ";
6360 0 : cmd += fSql->QuoteString(xtable_name.c_str());
6361 0 : cmd += ", ";
6362 : char buf[256];
6363 0 : sprintf(buf, "%.0f", (double)timestamp);
6364 0 : cmd += buf;
6365 0 : cmd += ", ";
6366 0 : cmd += fSql->QuoteString("1");
6367 0 : cmd += ");";
6368 :
6369 0 : int status = fSql->Exec(table_name.c_str(), cmd.c_str());
6370 0 : if (status == DB_SUCCESS)
6371 0 : break;
6372 :
6373 : // if INSERT failed _history_index does not exist then recover to savepoint t0
6374 : // to prevent whole transition abort
6375 0 : fSql->Exec(table_name.c_str(), "ROLLBACK TO SAVEPOINT t0");
6376 :
6377 0 : status = CreateSqlTable(fSql, "_history_index", &have_transaction, true);
6378 0 : status = CreateSqlColumn(fSql, "_history_index", "event_name", "text not null", &have_transaction, fDebug);
6379 0 : status = CreateSqlColumn(fSql, "_history_index", "table_name", "text", &have_transaction, fDebug);
6380 0 : status = CreateSqlColumn(fSql, "_history_index", "tag_name", "text", &have_transaction, fDebug);
6381 0 : status = CreateSqlColumn(fSql, "_history_index", "tag_type", "text", &have_transaction, fDebug);
6382 0 : status = CreateSqlColumn(fSql, "_history_index", "column_name", "text", &have_transaction, fDebug);
6383 0 : status = CreateSqlColumn(fSql, "_history_index", "column_type", "text", &have_transaction, fDebug);
6384 0 : status = CreateSqlColumn(fSql, "_history_index", "itimestamp", "integer not null", &have_transaction, fDebug);
6385 0 : status = CreateSqlColumn(fSql, "_history_index", "active", "smallint", &have_transaction, fDebug);
6386 :
6387 0 : status = fSql->CommitTransaction(table_name.c_str());
6388 0 : }
6389 :
6390 0 : if (status != DB_SUCCESS) {
6391 0 : return HS_FILE_ERROR;
6392 : }
6393 :
6394 0 : return ReadPgsqlTableNames(fSql, sv, xtable_name.c_str(), fDebug, event_name, xtable_name.c_str());
6395 0 : }
6396 :
6397 0 : cm_msg(MERROR, "PgsqlHistory::create_table", "Could not create table [%s] for event [%s], timestamp %s, after %d attempts", table_name.c_str(), event_name, TimeToString(timestamp).c_str(), max_attempts);
6398 :
6399 0 : return HS_FILE_ERROR;
6400 0 : }
6401 :
6402 0 : int PgsqlHistory::update_column(const char* event_name, const char* table_name, const char* column_name, const char* column_type, const char* tag_name, const char* tag_type, const time_t timestamp, bool active, bool* have_transaction)
6403 : {
6404 0 : if (fDebug)
6405 0 : printf("PgsqlHistory::update_column: event [%s], table [%s], column [%s], type [%s] new name [%s], timestamp %s\n", event_name, table_name, column_name, column_type, tag_name, TimeToString(timestamp).c_str());
6406 :
6407 0 : std::string cmd;
6408 0 : cmd += "INSERT INTO _history_index (event_name, table_name, tag_name, tag_type, column_name, column_type, itimestamp, active) VALUES (";
6409 0 : cmd += fSql->QuoteString(event_name);
6410 0 : cmd += ", ";
6411 0 : cmd += fSql->QuoteString(table_name);
6412 0 : cmd += ", ";
6413 0 : cmd += fSql->QuoteString(tag_name);
6414 0 : cmd += ", ";
6415 0 : cmd += fSql->QuoteString(tag_type);
6416 0 : cmd += ", ";
6417 0 : cmd += fSql->QuoteString(column_name);
6418 0 : cmd += ", ";
6419 0 : cmd += fSql->QuoteString(column_type);
6420 0 : cmd += ", ";
6421 : char buf[256];
6422 0 : sprintf(buf, "%.0f", (double)timestamp);
6423 0 : cmd += buf;
6424 0 : cmd += ", ";
6425 0 : if (active)
6426 0 : cmd += fSql->QuoteString("1");
6427 : else
6428 0 : cmd += fSql->QuoteString("0");
6429 0 : cmd += ");";
6430 :
6431 0 : int status = fSql->Exec(table_name, cmd.c_str());
6432 0 : if (status != DB_SUCCESS)
6433 0 : return HS_FILE_ERROR;
6434 :
6435 0 : return HS_SUCCESS;
6436 0 : }
6437 :
6438 : #endif // HAVE_PGSQL
6439 :
6440 : ////////////////////////////////////////////////////////
6441 : // File history class //
6442 : ////////////////////////////////////////////////////////
6443 :
6444 : const time_t kDay = 24*60*60;
6445 : const time_t kMonth = 30*kDay;
6446 :
6447 : const double KiB = 1024;
6448 : const double MiB = KiB*KiB;
6449 : //const double GiB = KiB*MiB;
6450 :
6451 : class FileHistory: public SchemaHistoryBase
6452 : {
6453 : protected:
6454 : std::string fPath;
6455 : time_t fPathLastMtime = 0;
6456 : std::vector<std::string> fSortedFiles;
6457 : std::vector<bool> fSortedRead;
6458 : time_t fConfMaxFileAge = 1*kMonth;
6459 : off64_t fConfMaxFileSize = 100*MiB;
6460 :
6461 : public:
6462 0 : FileHistory() // ctor
6463 0 : {
6464 : // empty
6465 0 : }
6466 :
6467 : int hs_connect(const char* connect_string);
6468 : int hs_disconnect();
6469 : int hs_clear_cache();
6470 : int read_schema(HsSchemaVector* sv, const char* event_name, const time_t timestamp);
6471 : HsSchema* new_event(const char* event_name, time_t timestamp, int ntags, const TAG tags[]);
6472 :
6473 : protected:
6474 : int create_file(const char* event_name, time_t timestamp, const std::vector<HsSchemaEntry>& vars, std::string* filenamep);
6475 : HsFileSchema* read_file_schema(const char* filename);
6476 : int read_file_list(bool *pchanged);
6477 : void clear_file_list();
6478 : void tags_to_variables(int ntags, const TAG tags[], std::vector<HsSchemaEntry>& variables);
6479 : HsSchema* maybe_reopen(const char* event_name, time_t timestamp, HsSchema* s);
6480 : };
6481 :
6482 0 : int FileHistory::hs_connect(const char* connect_string)
6483 : {
6484 0 : if (fDebug)
6485 0 : printf("hs_connect [%s]!\n", connect_string);
6486 :
6487 0 : hs_disconnect();
6488 :
6489 0 : fConnectString = connect_string;
6490 0 : fPath = connect_string;
6491 :
6492 : // add trailing '/'
6493 0 : if (fPath.length() > 0) {
6494 0 : if (fPath[fPath.length()-1] != DIR_SEPARATOR)
6495 0 : fPath += DIR_SEPARATOR_STR;
6496 : }
6497 :
6498 0 : return HS_SUCCESS;
6499 : }
6500 :
6501 0 : int FileHistory::hs_clear_cache()
6502 : {
6503 0 : if (fDebug)
6504 0 : printf("FileHistory::hs_clear_cache!\n");
6505 0 : fPathLastMtime = 0;
6506 0 : return SchemaHistoryBase::hs_clear_cache();
6507 : }
6508 :
6509 0 : int FileHistory::hs_disconnect()
6510 : {
6511 0 : if (fDebug)
6512 0 : printf("FileHistory::hs_disconnect!\n");
6513 :
6514 0 : hs_flush_buffers();
6515 0 : hs_clear_cache();
6516 :
6517 0 : return HS_SUCCESS;
6518 : }
6519 :
6520 0 : void FileHistory::clear_file_list()
6521 : {
6522 0 : fPathLastMtime = 0;
6523 0 : fSortedFiles.clear();
6524 0 : fSortedRead.clear();
6525 0 : }
6526 :
6527 0 : int FileHistory::read_file_list(bool* pchanged)
6528 : {
6529 : int status;
6530 0 : double start_time = ss_time_sec();
6531 :
6532 0 : if (pchanged)
6533 0 : *pchanged = false;
6534 :
6535 : struct stat stat_buf;
6536 0 : status = stat(fPath.c_str(), &stat_buf);
6537 0 : if (status != 0) {
6538 0 : cm_msg(MERROR, "FileHistory::read_file_list", "Cannot stat(%s), errno %d (%s)", fPath.c_str(), errno, strerror(errno));
6539 0 : return HS_FILE_ERROR;
6540 : }
6541 :
6542 : //printf("dir [%s], mtime: %d %d last: %d %d, mtime %s", fPath.c_str(), stat_buf.st_mtimespec.tv_sec, stat_buf.st_mtimespec.tv_nsec, last_mtimespec.tv_sec, last_mtimespec.tv_nsec, ctime(&stat_buf.st_mtimespec.tv_sec));
6543 :
6544 0 : if (stat_buf.st_mtime == fPathLastMtime) {
6545 0 : if (fDebug)
6546 0 : printf("FileHistory::read_file_list: history directory \"%s\" mtime %d did not change\n", fPath.c_str(), int(stat_buf.st_mtime));
6547 0 : return HS_SUCCESS;
6548 : }
6549 :
6550 0 : fPathLastMtime = stat_buf.st_mtime;
6551 :
6552 0 : if (fDebug)
6553 0 : printf("FileHistory::read_file_list: reading list of history files in \"%s\"\n", fPath.c_str());
6554 :
6555 0 : std::vector<std::string> flist;
6556 :
6557 0 : ss_file_find(fPath.c_str(), "mhf_*.dat", &flist);
6558 :
6559 0 : double ls_time = ss_time_sec();
6560 0 : double ls_elapsed = ls_time - start_time;
6561 0 : if (ls_elapsed > 5.000) {
6562 0 : cm_msg(MINFO, "FileHistory::read_file_list", "\"ls -l\" of \"%s\" took %.1f sec", fPath.c_str(), ls_elapsed);
6563 0 : cm_msg_flush_buffer();
6564 : }
6565 :
6566 : // note: reverse iterator is used to sort filenames by time, newest first
6567 0 : std::sort(flist.rbegin(), flist.rend());
6568 :
6569 : #if 0
6570 : {
6571 : printf("file names sorted by time:\n");
6572 : for (size_t i=0; i<flist.size(); i++) {
6573 : printf("%d: %s\n", i, flist[i].c_str());
6574 : }
6575 : }
6576 : #endif
6577 :
6578 0 : std::vector<bool> fread;
6579 0 : fread.resize(flist.size()); // fill with "false"
6580 :
6581 : // loop over the old list of files,
6582 : // for files we already read, loop over new file
6583 : // list and mark the same file as read. K.O.
6584 0 : for (size_t j=0; j<fSortedFiles.size(); j++) {
6585 0 : if (fSortedRead[j]) {
6586 0 : for (size_t i=0; i<flist.size(); i++) {
6587 0 : if (flist[i] == fSortedFiles[j]) {
6588 0 : fread[i] = true;
6589 0 : break;
6590 : }
6591 : }
6592 : }
6593 : }
6594 :
6595 0 : fSortedFiles = flist;
6596 0 : fSortedRead = fread;
6597 :
6598 0 : if (pchanged)
6599 0 : *pchanged = true;
6600 :
6601 0 : return HS_SUCCESS;
6602 0 : }
6603 :
6604 0 : int FileHistory::read_schema(HsSchemaVector* sv, const char* event_name, const time_t timestamp)
6605 : {
6606 0 : if (fDebug)
6607 0 : printf("FileHistory::read_schema: event [%s] at time %s\n", event_name, TimeToString(timestamp).c_str());
6608 :
6609 0 : if (sv->size() == 0) {
6610 0 : if (fDebug)
6611 0 : printf("FileHistory::read_schema: schema is empty, do a full reload from disk\n");
6612 0 : clear_file_list();
6613 : }
6614 :
6615 0 : BOOL old_call_watchdog = FALSE;
6616 0 : DWORD old_timeout = 0;
6617 0 : cm_get_watchdog_params(&old_call_watchdog, &old_timeout);
6618 0 : cm_set_watchdog_params(old_call_watchdog, 0);
6619 :
6620 0 : bool changed = false;
6621 :
6622 0 : int status = read_file_list(&changed);
6623 :
6624 0 : if (status != HS_SUCCESS) {
6625 0 : cm_set_watchdog_params(old_call_watchdog, old_timeout);
6626 0 : return status;
6627 : }
6628 :
6629 0 : if (!changed) {
6630 0 : if ((*sv).find_event(event_name, timestamp)) {
6631 0 : if (fDebug)
6632 0 : printf("FileHistory::read_schema: event [%s] at time %s, no new history files, already have this schema\n", event_name, TimeToString(timestamp).c_str());
6633 0 : cm_set_watchdog_params(old_call_watchdog, old_timeout);
6634 0 : return HS_SUCCESS;
6635 : }
6636 : }
6637 :
6638 0 : double start_time = ss_time_sec();
6639 :
6640 0 : int count_read = 0;
6641 :
6642 0 : for (size_t i=0; i<fSortedFiles.size(); i++) {
6643 0 : std::string file_name = fPath + fSortedFiles[i];
6644 0 : if (fSortedRead[i])
6645 0 : continue;
6646 : //bool dupe = false;
6647 : //for (size_t ss=0; ss<sv->size(); ss++) {
6648 : // HsFileSchema* ssp = (HsFileSchema*)(*sv)[ss];
6649 : // if (file_name == ssp->fFileName) {
6650 : // dupe = true;
6651 : // break;
6652 : // }
6653 : //}
6654 : //if (dupe)
6655 : // continue;
6656 0 : fSortedRead[i] = true;
6657 0 : HsFileSchema* s = read_file_schema(file_name.c_str());
6658 0 : if (!s)
6659 0 : continue;
6660 0 : sv->add(s);
6661 0 : count_read++;
6662 :
6663 0 : if (event_name) {
6664 0 : if (s->fEventName == event_name) {
6665 : //printf("file %s event_name %s time %s, age %f\n", file_name.c_str(), s->fEventName.c_str(), TimeToString(s->fTimeFrom).c_str(), double(timestamp - s->fTimeFrom));
6666 0 : if (s->fTimeFrom <= timestamp) {
6667 : // this file is older than the time requested,
6668 : // subsequent files will be even older,
6669 : // we can stop reading here.
6670 0 : break;
6671 : }
6672 : }
6673 : }
6674 0 : }
6675 :
6676 0 : double end_time = ss_time_sec();
6677 0 : double read_elapsed = end_time - start_time;
6678 0 : if (read_elapsed > 5.000) {
6679 0 : cm_msg(MINFO, "FileHistory::read_schema", "Loading schema for event \"%s\" timestamp %s, reading %d history files took %.1f sec", event_name, TimeToString(timestamp).c_str(), count_read, read_elapsed);
6680 0 : cm_msg_flush_buffer();
6681 : }
6682 :
6683 0 : cm_set_watchdog_params(old_call_watchdog, old_timeout);
6684 :
6685 0 : return HS_SUCCESS;
6686 : }
6687 :
6688 0 : void FileHistory::tags_to_variables(int ntags, const TAG tags[], std::vector<HsSchemaEntry>& variables)
6689 : {
6690 0 : for (int i=0; i<ntags; i++) {
6691 0 : HsSchemaEntry e;
6692 :
6693 0 : int tsize = rpc_tid_size(tags[i].type);
6694 :
6695 0 : e.tag_name = tags[i].name;
6696 0 : e.tag_type = rpc_tid_name(tags[i].type);
6697 0 : e.name = tags[i].name;
6698 0 : e.type = tags[i].type;
6699 0 : e.n_data = tags[i].n_data;
6700 0 : e.n_bytes = tags[i].n_data*tsize;
6701 :
6702 0 : variables.push_back(e);
6703 0 : }
6704 0 : }
6705 :
6706 0 : HsSchema* FileHistory::new_event(const char* event_name, time_t timestamp, int ntags, const TAG tags[])
6707 : {
6708 0 : if (fDebug)
6709 0 : printf("FileHistory::new_event: event [%s], timestamp %s, ntags %d\n", event_name, TimeToString(timestamp).c_str(), ntags);
6710 :
6711 : int status;
6712 :
6713 0 : HsFileSchema* s = (HsFileSchema*)fWriterSchema.find_event(event_name, timestamp);
6714 :
6715 0 : if (!s) {
6716 : //printf("hs_define_event: no schema for event %s\n", event_name);
6717 0 : status = read_schema(&fWriterSchema, event_name, timestamp);
6718 0 : if (status != HS_SUCCESS)
6719 0 : return NULL;
6720 0 : s = (HsFileSchema*)fWriterSchema.find_event(event_name, timestamp);
6721 : } else {
6722 : //printf("hs_define_event: already have schema for event %s\n", s->fEventName.c_str());
6723 : }
6724 :
6725 0 : bool xdebug = false;
6726 :
6727 0 : if (s) { // is existing schema the same as new schema?
6728 0 : bool same = true;
6729 :
6730 0 : if (same) {
6731 0 : if (s->fEventName != event_name) {
6732 0 : if (xdebug)
6733 0 : printf("AAA: [%s] [%s]!\n", s->fEventName.c_str(), event_name);
6734 0 : same = false;
6735 : }
6736 : }
6737 :
6738 0 : if (same) {
6739 0 : if (s->fVariables.size() != (size_t)ntags) {
6740 0 : if (xdebug)
6741 0 : printf("BBB: event [%s]: ntags: %zu -> %d!\n", event_name, s->fVariables.size(), ntags);
6742 0 : same = false;
6743 : }
6744 : }
6745 :
6746 0 : if (same) {
6747 0 : for (size_t i=0; i<s->fVariables.size(); i++) {
6748 0 : if (s->fVariables[i].name != tags[i].name) {
6749 0 : if (xdebug)
6750 0 : printf("CCC: event [%s] index %zu: name [%s] -> [%s]!\n", event_name, i, s->fVariables[i].name.c_str(), tags[i].name);
6751 0 : same = false;
6752 : }
6753 0 : if (s->fVariables[i].type != (int)tags[i].type) {
6754 0 : if (xdebug)
6755 0 : printf("DDD: event [%s] index %zu: type %d -> %d!\n", event_name, i, s->fVariables[i].type, tags[i].type);
6756 0 : same = false;
6757 : }
6758 0 : if (s->fVariables[i].n_data != (int)tags[i].n_data) {
6759 0 : if (xdebug)
6760 0 : printf("EEE: event [%s] index %zu: n_data %d -> %d!\n", event_name, i, s->fVariables[i].n_data, tags[i].n_data);
6761 0 : same = false;
6762 : }
6763 0 : if (!same)
6764 0 : break;
6765 : }
6766 : }
6767 :
6768 0 : if (!same) {
6769 0 : if (xdebug) {
6770 0 : printf("*** Schema for event %s has changed!\n", event_name);
6771 :
6772 0 : printf("*** Old schema for event [%s] time %s:\n", event_name, TimeToString(timestamp).c_str());
6773 0 : s->print();
6774 0 : printf("*** New tags:\n");
6775 0 : PrintTags(ntags, tags);
6776 : }
6777 :
6778 0 : if (fDebug)
6779 0 : printf("FileHistory::new_event: event [%s], timestamp %s, ntags %d: schema mismatch, starting a new file.\n", event_name, TimeToString(timestamp).c_str(), ntags);
6780 :
6781 0 : s = NULL;
6782 : }
6783 : }
6784 :
6785 0 : if (s) {
6786 : // maybe this schema is too old - rotate files every so often
6787 0 : time_t age = timestamp - s->fTimeFrom;
6788 : //printf("*** age %s (%.1f months), timestamp %s, time_from %s, file %s\n", TimeToString(age).c_str(), (double)age/(double)kMonth, TimeToString(timestamp).c_str(), TimeToString(s->fTimeFrom).c_str(), s->fFileName.c_str());
6789 0 : if (age > fConfMaxFileAge) {
6790 0 : if (fDebug)
6791 0 : printf("FileHistory::new_event: event [%s], timestamp %s, ntags %d: schema is too old, age %.1f months, starting a new file.\n", event_name, TimeToString(timestamp).c_str(), ntags, (double)age/(double)kMonth);
6792 :
6793 : // force creation of a new file
6794 0 : s = NULL;
6795 : }
6796 : }
6797 :
6798 0 : if (s) {
6799 : // maybe this file is too big - rotate files to limit maximum size
6800 0 : double size = ss_file_size(s->fFileName.c_str());
6801 : //printf("*** size %.0f, file %s\n", size, s->fFileName.c_str());
6802 0 : if (size > fConfMaxFileSize) {
6803 0 : if (fDebug)
6804 0 : printf("FileHistory::new_event: event [%s], timestamp %s, ntags %d: file too big, size %.1f MiBytes, max size %.1f MiBytes, starting a new file.\n", event_name, TimeToString(timestamp).c_str(), ntags, size/MiB, fConfMaxFileSize/MiB);
6805 :
6806 : // force creation of a new file
6807 0 : s = NULL;
6808 : }
6809 : }
6810 :
6811 0 : if (!s) {
6812 0 : std::string filename;
6813 :
6814 0 : std::vector<HsSchemaEntry> vars;
6815 :
6816 0 : tags_to_variables(ntags, tags, vars);
6817 :
6818 0 : status = create_file(event_name, timestamp, vars, &filename);
6819 0 : if (status != HS_SUCCESS)
6820 0 : return NULL;
6821 :
6822 0 : HsFileSchema* ss = read_file_schema(filename.c_str());
6823 0 : if (!ss) {
6824 0 : cm_msg(MERROR, "FileHistory::new_event", "Error: Cannot create schema for event \'%s\', see previous messages", event_name);
6825 0 : return NULL;
6826 : }
6827 :
6828 0 : fWriterSchema.add(ss);
6829 :
6830 0 : s = (HsFileSchema*)fWriterSchema.find_event(event_name, timestamp);
6831 :
6832 0 : if (!s) {
6833 0 : cm_msg(MERROR, "FileHistory::new_event", "Error: Cannot create schema for event \'%s\', see previous messages", event_name);
6834 0 : return NULL;
6835 : }
6836 :
6837 0 : if (xdebug) {
6838 0 : printf("*** New schema for event [%s] time %s:\n", event_name, TimeToString(timestamp).c_str());
6839 0 : s->print();
6840 : }
6841 0 : }
6842 :
6843 0 : assert(s != NULL);
6844 :
6845 : #if 0
6846 : {
6847 : printf("schema for [%s] is %p\n", event_name, s);
6848 : if (s)
6849 : s->print();
6850 : }
6851 : #endif
6852 :
6853 0 : HsFileSchema* e = new HsFileSchema();
6854 :
6855 0 : *e = *s; // make a copy of the schema
6856 :
6857 0 : return e;
6858 : }
6859 :
6860 0 : int FileHistory::create_file(const char* event_name, time_t timestamp, const std::vector<HsSchemaEntry>& vars, std::string* filenamep)
6861 : {
6862 0 : if (fDebug)
6863 0 : printf("FileHistory::create_file: event [%s]\n", event_name);
6864 :
6865 : // NB: file names are constructed in such a way
6866 : // that when sorted lexicographically ("ls -1 | sort")
6867 : // they *also* become sorted by time
6868 :
6869 : struct tm tm;
6870 0 : localtime_r(×tamp, &tm); // somebody must call tzset() before this.
6871 :
6872 : char buf[256];
6873 0 : strftime(buf, sizeof(buf), "%Y%m%d", &tm);
6874 :
6875 0 : std::string filename;
6876 0 : filename += fPath;
6877 0 : filename += "mhf_";
6878 0 : filename += TimeToString(timestamp);
6879 0 : filename += "_";
6880 0 : filename += buf;
6881 0 : filename += "_";
6882 0 : filename += MidasNameToFileName(event_name);
6883 :
6884 0 : std::string try_filename = filename + ".dat";
6885 :
6886 0 : FILE *fp = NULL;
6887 0 : for (int itry=0; itry<10; itry++) {
6888 0 : if (itry > 0) {
6889 : char s[256];
6890 0 : sprintf(s, "_%d", rand());
6891 0 : try_filename = filename + s + ".dat";
6892 : }
6893 :
6894 0 : fp = fopen(try_filename.c_str(), "r");
6895 0 : if (fp != NULL) {
6896 : // this file already exists, try with a different name
6897 0 : fclose(fp);
6898 0 : continue;
6899 : }
6900 :
6901 0 : fp = fopen(try_filename.c_str(), "w");
6902 0 : if (fp == NULL) {
6903 : // somehow cannot create this file, try again
6904 0 : cm_msg(MERROR, "FileHistory::create_file", "Error: Cannot create file \'%s\' for event \'%s\', fopen() errno %d (%s)", try_filename.c_str(), event_name, errno, strerror(errno));
6905 0 : continue;
6906 : }
6907 :
6908 : // file opened
6909 0 : break;
6910 : }
6911 :
6912 0 : if (fp == NULL) {
6913 : // somehow cannot create any file, whine!
6914 0 : cm_msg(MERROR, "FileHistory::create_file", "Error: Cannot create file \'%s\' for event \'%s\'", filename.c_str(), event_name);
6915 0 : return HS_FILE_ERROR;
6916 : }
6917 :
6918 0 : std::string ss;
6919 :
6920 0 : ss += "version: 2.0\n";
6921 0 : ss += "event_name: ";
6922 0 : ss += event_name;
6923 0 : ss += "\n";
6924 0 : ss += "time: ";
6925 0 : ss += TimeToString(timestamp);
6926 0 : ss += "\n";
6927 :
6928 0 : int recsize = 0;
6929 :
6930 0 : ss += "tag: /DWORD 1 4 /timestamp\n";
6931 0 : recsize += 4;
6932 :
6933 0 : bool padded = false;
6934 0 : int offset = 0;
6935 :
6936 0 : bool xdebug = false; // (strcmp(event_name, "u_Beam") == 0);
6937 :
6938 0 : for (size_t i=0; i<vars.size(); i++) {
6939 0 : int tsize = rpc_tid_size(vars[i].type);
6940 0 : int n_bytes = vars[i].n_data*tsize;
6941 0 : int xalign = (offset % tsize);
6942 :
6943 0 : if (xdebug)
6944 0 : printf("tag %zu, tsize %d, n_bytes %d, xalign %d\n", i, tsize, n_bytes, xalign);
6945 :
6946 : #if 0
6947 : // looks like history data does not do alignement and padding
6948 : if (xalign != 0) {
6949 : padded = true;
6950 : int pad_bytes = tsize - xalign;
6951 : assert(pad_bytes > 0);
6952 :
6953 : ss += "tag: ";
6954 : ss += "XPAD";
6955 : ss += " ";
6956 : ss += SmallIntToString(1);
6957 : ss += " ";
6958 : ss += SmallIntToString(pad_bytes);
6959 : ss += " ";
6960 : ss += "pad_";
6961 : ss += SmallIntToString(i);
6962 : ss += "\n";
6963 :
6964 : offset += pad_bytes;
6965 : recsize += pad_bytes;
6966 :
6967 : assert((offset % tsize) == 0);
6968 : fprintf(stderr, "FIXME: need to debug padding!\n");
6969 : //abort();
6970 : }
6971 : #endif
6972 :
6973 0 : ss += "tag: ";
6974 0 : ss += rpc_tid_name(vars[i].type);
6975 0 : ss += " ";
6976 0 : ss += SmallIntToString(vars[i].n_data);
6977 0 : ss += " ";
6978 0 : ss += SmallIntToString(n_bytes);
6979 0 : ss += " ";
6980 0 : ss += vars[i].name;
6981 0 : ss += "\n";
6982 :
6983 0 : recsize += n_bytes;
6984 0 : offset += n_bytes;
6985 : }
6986 :
6987 0 : ss += "record_size: ";
6988 0 : ss += SmallIntToString(recsize);
6989 0 : ss += "\n";
6990 :
6991 : // reserve space for "data_offset: ..."
6992 0 : int sslength = ss.length() + 127;
6993 :
6994 0 : int block = 1024;
6995 0 : int nb = (sslength + block - 1)/block;
6996 0 : int data_offset = block * nb;
6997 :
6998 0 : ss += "data_offset: ";
6999 0 : ss += SmallIntToString(data_offset);
7000 0 : ss += "\n";
7001 :
7002 0 : fprintf(fp, "%s", ss.c_str());
7003 :
7004 0 : fclose(fp);
7005 :
7006 0 : if (1 && padded) {
7007 0 : printf("Schema in file %s has padding:\n", try_filename.c_str());
7008 0 : printf("%s", ss.c_str());
7009 : }
7010 :
7011 0 : if (filenamep)
7012 0 : *filenamep = try_filename;
7013 :
7014 0 : return HS_SUCCESS;
7015 0 : }
7016 :
7017 0 : HsFileSchema* FileHistory::read_file_schema(const char* filename)
7018 : {
7019 0 : if (fDebug)
7020 0 : printf("FileHistory::read_file_schema: file %s\n", filename);
7021 :
7022 0 : FILE* fp = fopen(filename, "r");
7023 0 : if (!fp) {
7024 0 : cm_msg(MERROR, "FileHistory::read_file_schema", "Cannot read \'%s\', fopen() errno %d (%s)", filename, errno, strerror(errno));
7025 0 : return NULL;
7026 : }
7027 :
7028 0 : HsFileSchema* s = NULL;
7029 :
7030 : // File format looks like this:
7031 : // version: 2.0
7032 : // event_name: u_Beam
7033 : // time: 1023174012
7034 : // tag: /DWORD 1 4 /timestamp
7035 : // tag: FLOAT 1 4 B1
7036 : // ...
7037 : // tag: FLOAT 1 4 Ref Heater
7038 : // record_size: 84
7039 : // data_offset: 1024
7040 :
7041 0 : size_t rd_recsize = 0;
7042 0 : int offset = 0;
7043 :
7044 : while (1) {
7045 : char buf[1024];
7046 0 : char* b = fgets(buf, sizeof(buf), fp);
7047 :
7048 : //printf("read: %s\n", b);
7049 :
7050 0 : if (!b) {
7051 0 : break; // end of file
7052 : }
7053 :
7054 : char*bb;
7055 :
7056 0 : bb = strchr(b, '\n');
7057 0 : if (bb)
7058 0 : *bb = 0;
7059 :
7060 0 : bb = strchr(b, '\r');
7061 0 : if (bb)
7062 0 : *bb = 0;
7063 :
7064 0 : bb = strstr(b, "version: 2.0");
7065 0 : if (bb == b) {
7066 0 : s = new HsFileSchema();
7067 0 : assert(s);
7068 :
7069 0 : s->fFileName = filename;
7070 0 : continue;
7071 : }
7072 :
7073 0 : if (!s) {
7074 : // malformed history file
7075 0 : break;
7076 : }
7077 :
7078 0 : bb = strstr(b, "event_name: ");
7079 0 : if (bb == b) {
7080 0 : s->fEventName = bb + 12;
7081 0 : continue;
7082 : }
7083 :
7084 0 : bb = strstr(b, "time: ");
7085 0 : if (bb == b) {
7086 0 : s->fTimeFrom = strtoul(bb + 6, NULL, 10);
7087 0 : continue;
7088 : }
7089 :
7090 : // tag format is like this:
7091 : //
7092 : // tag: FLOAT 1 4 Ref Heater
7093 : //
7094 : // "FLOAT" is the MIDAS type, "/DWORD" is special tag for the timestamp
7095 : // "1" is the number of array elements
7096 : // "4" is the total tag size in bytes (n_data*tid_size)
7097 : // "Ref Heater" is the tag name
7098 :
7099 0 : bb = strstr(b, "tag: ");
7100 0 : if (bb == b) {
7101 0 : bb += 5; // now points to the tag MIDAS type
7102 0 : const char* midas_type = bb;
7103 0 : char* bbb = strchr(bb, ' ');
7104 0 : if (bbb) {
7105 0 : *bbb = 0;
7106 0 : HsSchemaEntry t;
7107 0 : if (midas_type[0] == '/') {
7108 0 : t.type = 0;
7109 : } else {
7110 0 : t.type = rpc_name_tid(midas_type);
7111 0 : if (t.type == 0) {
7112 0 : cm_msg(MERROR, "FileHistory::read_file_schema", "Unknown MIDAS data type \'%s\' in history file \'%s\'", midas_type, filename);
7113 0 : if (s)
7114 0 : delete s;
7115 0 : s = NULL;
7116 0 : break;
7117 : }
7118 : }
7119 0 : bbb++;
7120 0 : while (*bbb == ' ')
7121 0 : bbb++;
7122 0 : if (*bbb) {
7123 0 : t.n_data = strtoul(bbb, &bbb, 10);
7124 0 : while (*bbb == ' ')
7125 0 : bbb++;
7126 0 : if (*bbb) {
7127 0 : t.n_bytes = strtoul(bbb, &bbb, 10);
7128 0 : while (*bbb == ' ')
7129 0 : bbb++;
7130 0 : t.name = bbb;
7131 : }
7132 : }
7133 :
7134 0 : if (midas_type[0] != '/') {
7135 0 : s->fVariables.push_back(t);
7136 0 : s->fOffsets.push_back(offset);
7137 0 : offset += t.n_bytes;
7138 : }
7139 :
7140 0 : rd_recsize += t.n_bytes;
7141 0 : }
7142 0 : continue;
7143 0 : }
7144 :
7145 0 : bb = strstr(b, "record_size: ");
7146 0 : if (bb == b) {
7147 0 : s->fRecordSize = atoi(bb + 12);
7148 0 : continue;
7149 : }
7150 :
7151 0 : bb = strstr(b, "data_offset: ");
7152 0 : if (bb == b) {
7153 0 : s->fDataOffset = atoi(bb + 12);
7154 : // data offset is the last entry in the file
7155 0 : break;
7156 : }
7157 0 : }
7158 :
7159 0 : fclose(fp);
7160 :
7161 0 : if (!s) {
7162 0 : cm_msg(MERROR, "FileHistory::read_file_schema", "Malformed history schema in \'%s\', maybe it is not a history file", filename);
7163 0 : return NULL;
7164 : }
7165 :
7166 0 : if (rd_recsize != s->fRecordSize) {
7167 0 : cm_msg(MERROR, "FileHistory::read_file_schema", "Record size mismatch in history schema from \'%s\', file says %zu while total of all tags is %zu", filename, s->fRecordSize, rd_recsize);
7168 0 : if (s)
7169 0 : delete s;
7170 0 : return NULL;
7171 : }
7172 :
7173 0 : if (!s) {
7174 0 : cm_msg(MERROR, "FileHistory::read_file_schema", "Could not read history schema from \'%s\', maybe it is not a history file", filename);
7175 0 : if (s)
7176 0 : delete s;
7177 0 : return NULL;
7178 : }
7179 :
7180 0 : if (fDebug > 1)
7181 0 : s->print();
7182 :
7183 0 : return s;
7184 : }
7185 :
7186 0 : HsSchema* FileHistory::maybe_reopen(const char* event_name, time_t timestamp, HsSchema* s)
7187 : {
7188 0 : HsFileSchema* fs = dynamic_cast<HsFileSchema*>(s);
7189 :
7190 0 : assert(fs != NULL); // FileHistory::maybe_reopen() must be called only with file history schema.
7191 :
7192 0 : if (fs->fFileSize <= fConfMaxFileSize) {
7193 : // not big enough, let it grow
7194 0 : return s;
7195 : }
7196 :
7197 : // must rotate the file
7198 :
7199 0 : if (fDebug) {
7200 : #ifdef OS_DARWIN
7201 : printf("FileHistory::maybe_reopen: reopen file \"%s\", size %lld, max size %lld\n", fs->fFileName.c_str(), fs->fFileSize, fConfMaxFileSize);
7202 : #else
7203 0 : printf("FileHistory::maybe_reopen: reopen file \"%s\", size %jd, max size %jd\n", fs->fFileName.c_str(), fs->fFileSize, fConfMaxFileSize);
7204 : #endif
7205 : }
7206 :
7207 0 : std::string new_filename;
7208 :
7209 0 : int status = create_file(event_name, timestamp, fs->fVariables, &new_filename);
7210 :
7211 0 : if (status != HS_SUCCESS) {
7212 : // cannot create new history file
7213 0 : return s;
7214 : }
7215 :
7216 0 : HsFileSchema* new_fs = read_file_schema(new_filename.c_str());
7217 :
7218 0 : if (!new_fs) {
7219 : // cannot open new history file
7220 0 : return s;
7221 : }
7222 :
7223 0 : new_fs->fDisabled = false;
7224 :
7225 0 : HsFileSchema* new_fs_copy = new HsFileSchema;
7226 0 : *new_fs_copy = *new_fs; // make a copy
7227 :
7228 : //printf("replacing schema %p %p with %p %p\n", s, fs, new_fs, new_fs_copy);
7229 :
7230 0 : for (size_t i=0; i<fWriterEvents.size(); i++) {
7231 0 : if (s == fWriterEvents[i]) {
7232 0 : delete fWriterEvents[i];
7233 0 : fWriterEvents[i] = new_fs;
7234 0 : s = NULL; // pointer to fEvents[i]
7235 0 : fs = NULL; // pointer to fEvents[i]
7236 : }
7237 : }
7238 :
7239 0 : assert(s == NULL); // the schema we are replacing must be in fWriterEvents.
7240 :
7241 0 : fWriterSchema.add(new_fs_copy); // make sure new file is added to the list of files
7242 :
7243 0 : assert(new_fs->fFileSize < fConfMaxFileSize); // check that we are not returning the original big file
7244 :
7245 : //new_fs->print();
7246 :
7247 0 : return new_fs;
7248 0 : }
7249 :
7250 : ////////////////////////////////////////////////////////
7251 : // Factory constructors //
7252 : ////////////////////////////////////////////////////////
7253 :
7254 0 : MidasHistoryInterface* MakeMidasHistorySqlite()
7255 : {
7256 : #ifdef HAVE_SQLITE
7257 0 : return new SqliteHistory();
7258 : #else
7259 : cm_msg(MERROR, "MakeMidasHistorySqlite", "Error: Cannot initialize SQLITE history - this MIDAS was built without SQLITE support - HAVE_SQLITE is not defined");
7260 : return NULL;
7261 : #endif
7262 : }
7263 :
7264 0 : MidasHistoryInterface* MakeMidasHistoryMysql()
7265 : {
7266 : #ifdef HAVE_MYSQL
7267 0 : return new MysqlHistory();
7268 : #else
7269 : cm_msg(MERROR, "MakeMidasHistoryMysql", "Error: Cannot initialize MySQL history - this MIDAS was built without MySQL support - HAVE_MYSQL is not defined");
7270 : return NULL;
7271 : #endif
7272 : }
7273 :
7274 0 : MidasHistoryInterface* MakeMidasHistoryPgsql()
7275 : {
7276 : #ifdef HAVE_PGSQL
7277 0 : return new PgsqlHistory();
7278 : #else
7279 : cm_msg(MERROR, "MakeMidasHistoryPgsql", "Error: Cannot initialize PgSQL history - this MIDAS was built without PostgreSQL support - HAVE_PGSQL is not defined");
7280 : return NULL;
7281 : #endif
7282 : }
7283 :
7284 0 : MidasHistoryInterface* MakeMidasHistoryFile()
7285 : {
7286 0 : return new FileHistory();
7287 : }
7288 :
7289 : /* emacs
7290 : * Local Variables:
7291 : * tab-width: 8
7292 : * c-basic-offset: 3
7293 : * indent-tabs-mode: nil
7294 : * End:
7295 : */
|
