Line data Source code
1 : //
2 : // mh2sql: import midas history into an SQL database
3 : //
4 : // Author: Konstantin Olchanski / TRIUMF, August-2008
5 : //
6 :
7 : #include <stdio.h>
8 : #include <stdlib.h>
9 : #include <stdint.h>
10 : #include <string.h>
11 : #include <assert.h>
12 : #include <errno.h>
13 :
14 : #include "midas.h"
15 : #include "history.h"
16 : #include "mstrlcpy.h"
17 :
18 : #include <map>
19 :
20 : std::map<int,const char*> gEventName;
21 :
22 : bool print_empty = false;
23 : bool print_dupe = false;
24 : bool print_string = false;
25 :
26 : bool had_empty = false;
27 : bool had_dupe = false;
28 : bool had_string = false;
29 :
30 : bool report_empty = true;
31 : bool report_dupe = true;
32 : bool report_string = true;
33 :
34 0 : int copyHstFile(const char* filename, FILE*f, MidasHistoryInterface *mh)
35 : {
36 0 : assert(f!=NULL);
37 :
38 0 : char *buf = NULL;
39 0 : int bufSize = 0;
40 :
41 : while (1)
42 : {
43 : HIST_RECORD rec;
44 :
45 0 : int rd = fread(&rec, sizeof(rec), 1, f);
46 0 : if (!rd)
47 0 : break;
48 :
49 : #if 0
50 : printf("HIST_RECORD:\n");
51 : printf(" Record type: 0x%x\n", rec.record_type);
52 : printf(" Event ID: %d\n", rec.event_id);
53 : printf(" Time: %d\n", rec.time);
54 : printf(" Offset: 0x%x\n", rec.def_offset);
55 : printf(" Size: 0x%x\n", rec.data_size);
56 : #endif
57 :
58 0 : switch (rec.record_type)
59 : {
60 0 : default:
61 0 : fprintf(stderr, "Error: %s: Unexpected record type: 0x%x\n", filename, rec.record_type);
62 0 : if (buf)
63 0 : free(buf);
64 0 : return -1;
65 : break;
66 :
67 0 : case 0x46445348: // RT_DEF:
68 : {
69 : char event_name[NAME_LENGTH+1];
70 0 : rd = fread(event_name, 1, NAME_LENGTH, f);
71 :
72 0 : if (rd != NAME_LENGTH) {
73 0 : fprintf(stderr, "Error: %s: Error reading RT_DEF record, fread(%d) returned %d, errno %d (%s)\n", filename, NAME_LENGTH, rd, errno, strerror(errno));
74 0 : if (buf)
75 0 : free(buf);
76 0 : return -1;
77 : break;
78 : }
79 :
80 : // make sure event name is NUL terminated
81 0 : event_name[NAME_LENGTH] = 0;
82 :
83 0 : if (strlen(event_name) == NAME_LENGTH) {
84 : //fprintf(stderr, "Warning: %s: Truncated event length in RT_DEF record: [%s]\n", filename, event_name);
85 : }
86 :
87 0 : int size = rec.data_size;
88 0 : int ntags = size/sizeof(TAG);
89 :
90 : // printf("Event %d, \"%s\", size %d, %d tags.\n", rec.event_id, event_name, size, ntags);
91 :
92 0 : if (!((size > 0) &&
93 : (size < 1*1024*1024) &&
94 0 : (size == ntags*(int)sizeof(TAG)))) {
95 :
96 0 : fprintf(stderr, "Error: %s: Invalid length of RT_DEF record: %d (0x%x)\n", filename, size, size);
97 0 : if (buf)
98 0 : free(buf);
99 0 : return -1;
100 : break;
101 : }
102 :
103 0 : TAG *tags = new TAG[ntags];
104 0 : rd = fread(tags, 1, size, f);
105 :
106 0 : if (rd != size) {
107 0 : fprintf(stderr, "Error: %s: Error reading RT_DEF record, fread(%d) returned %d, errno %d (%s)\n", filename, size, rd, errno, strerror(errno));
108 0 : if (buf)
109 0 : free(buf);
110 0 : return -1;
111 : break;
112 : }
113 :
114 : // need to sanitize the tag names
115 :
116 0 : for (int i=0; i<ntags; i++) {
117 0 : if (tags[i].type == TID_STRING) {
118 0 : if (print_string)
119 0 : fprintf(stderr, "Warning: %s: Event \"%s\": Fixed by truncation at forbidden TID_STRING tag \"%s\" at index %d\n", filename, event_name, tags[i].name, i);
120 0 : ntags = i;
121 0 : had_string = true;
122 0 : break;
123 : }
124 :
125 0 : if (strlen(tags[i].name) < 1) {
126 0 : sprintf(tags[i].name, "empty_%d", i);
127 0 : if (print_empty)
128 0 : fprintf(stderr, "Warning: %s: Event \"%s\": Fixed empty tag name for tag %d: replaced with \"%s\"\n", filename, event_name, i, tags[i].name);
129 0 : had_empty = true;
130 : }
131 :
132 0 : for (int j=i+1; j<ntags; j++) {
133 0 : if (strcmp(tags[i].name, tags[j].name) == 0) {
134 : char str[256];
135 0 : sprintf(str, "_dup%d", j);
136 0 : mstrlcat(tags[j].name, str, sizeof(tags[j].name));
137 0 : if (print_dupe)
138 0 : fprintf(stderr, "Warning: %s: Event \"%s\": Fixed duplicate tag names: tag \"%s\" at index %d duplicate at index %d replaced with \"%s\"\n", filename, event_name, tags[i].name, i, j, tags[j].name);
139 0 : had_dupe = true;
140 : }
141 : }
142 : }
143 :
144 0 : mh->hs_define_event(event_name, rec.time, ntags, tags);
145 :
146 0 : gEventName[rec.event_id] = strdup(event_name);
147 :
148 0 : delete[] tags;
149 0 : break;
150 : }
151 :
152 0 : case 0x41445348: // RT_DATA:
153 : {
154 0 : int size = rec.data_size;
155 :
156 : if (0)
157 : printf("Data record, size %d.\n", size);
158 :
159 0 : if (!((size > 0) &&
160 : (size < 1*1024*1024))) {
161 :
162 0 : fprintf(stderr, "Error: %s: Invalid length of RT_DATA record: %d (0x%x)\n", filename, size, size);
163 0 : if (buf)
164 0 : free(buf);
165 0 : return -1;
166 : break;
167 : }
168 :
169 0 : if (size > bufSize)
170 : {
171 0 : bufSize = 1024*((size+1023)/1024);
172 0 : char *newbuf = (char*)realloc(buf, bufSize);
173 :
174 0 : if (newbuf == NULL) {
175 0 : fprintf(stderr, "Error: %s: Cannot realloc(%d), errno %d (%s)\n", filename, bufSize, errno, strerror(errno));
176 0 : if (buf)
177 0 : free(buf);
178 0 : return -1;
179 : break;
180 : }
181 :
182 0 : buf = newbuf;
183 : }
184 :
185 0 : rd = fread(buf, 1, size, f);
186 :
187 0 : if (rd != size) {
188 0 : fprintf(stderr, "Error: %s: Error reading RT_DATA record, fread(%d) returned %d, errno %d (%s)\n", filename, size, rd, errno, strerror(errno));
189 0 : if (buf)
190 0 : free(buf);
191 0 : return -1;
192 : break;
193 : }
194 :
195 0 : time_t t = (time_t)rec.time;
196 :
197 0 : mh->hs_write_event(gEventName[rec.event_id], t, size, buf);
198 :
199 0 : break;
200 : }
201 : }
202 :
203 0 : cm_msg_flush_buffer();
204 0 : }
205 :
206 0 : if (buf)
207 0 : free(buf);
208 :
209 0 : return 0;
210 : }
211 :
212 0 : int copyHst(const char* name, MidasHistoryInterface* mh)
213 : {
214 0 : FILE* f = fopen(name,"r");
215 0 : if (!f)
216 : {
217 0 : fprintf(stderr, "Error: Cannot open \'%s\', errno %d (%s)\n", name, errno, strerror(errno));
218 0 : return -1;
219 : }
220 :
221 0 : fprintf(stderr, "Reading %s\n", name);
222 0 : copyHstFile(name, f, mh);
223 0 : fclose(f);
224 0 : mh->hs_flush_buffers();
225 :
226 0 : if (had_empty && report_empty) {
227 0 : report_empty = false;
228 0 : fprintf(stderr, "Notice: Automatically repaired some empty tag names\n");
229 : }
230 :
231 0 : if (had_dupe && report_dupe) {
232 0 : report_dupe = false;
233 0 : fprintf(stderr, "Notice: Automatically repaired some duplicate tag names\n");
234 : }
235 :
236 0 : if (had_string && report_string) {
237 0 : report_string = false;
238 0 : fprintf(stderr, "Notice: Automatically truncated events that contain tags of forbidden type TID_STRING\n");
239 : }
240 :
241 0 : return 0;
242 : }
243 :
244 0 : void help()
245 : {
246 0 : fprintf(stderr,"Usage: mh2sql [-h] [switches...] file1.hst file2.hst ...\n");
247 0 : fprintf(stderr,"\n");
248 0 : fprintf(stderr,"Switches:\n");
249 0 : fprintf(stderr," -h --- print this help message\n");
250 0 : fprintf(stderr," --report-repaired-tags --- print messages about all repaired empty and duplicate tag names\n");
251 0 : fprintf(stderr," --hs-debug <hs_debug_flag> --- set the history debug flag\n");
252 0 : fprintf(stderr," --odbc <ODBC_DSN> --- write to ODBC (SQL) history using given ODBC DSN\n");
253 0 : fprintf(stderr," --sqlite <path> --- write to SQLITE database at the given path\n");
254 0 : fprintf(stderr," --mysql mysql_writer.txt --- write to MYSQL database\n");
255 0 : fprintf(stderr," --pgsql pgsql_writer.txt --- write to PGSQL database\n");
256 0 : fprintf(stderr," --file <path> --- write to FILE database at the given path\n");
257 0 : fprintf(stderr,"\n");
258 0 : fprintf(stderr,"Examples:\n");
259 0 : fprintf(stderr," mh2sql --hs-debug 1 --sqlite . 130813.hst\n");
260 0 : exit(1);
261 : }
262 :
263 0 : int main(int argc,char*argv[])
264 : {
265 : int status;
266 0 : int hs_debug_flag = 0;
267 : HNDLE hDB;
268 0 : MidasHistoryInterface *mh = NULL;
269 :
270 0 : if (argc <= 2)
271 0 : help(); // DOES NOT RETURN
272 :
273 0 : status = cm_connect_experiment("", "", "mh2sql", NULL);
274 0 : if (status != SUCCESS)
275 0 : exit(1);
276 :
277 0 : status = cm_get_experiment_database(&hDB, NULL);
278 0 : assert(status == HS_SUCCESS);
279 :
280 : // turn off watchdog checks
281 0 : cm_set_watchdog_params(FALSE, 0);
282 :
283 0 : for (int iarg=1; iarg<argc; iarg++) {
284 0 : const char* arg = argv[iarg];
285 0 : if (arg[0] != '-')
286 0 : continue;
287 :
288 0 : if (strcmp(arg, "-h")==0) {
289 0 : help(); // DOES NOT RETURN
290 0 : } else if (strcmp(arg, "--report-repaired-tags") == 0) {
291 0 : print_empty = true;
292 0 : print_dupe = true;
293 0 : print_string = true;
294 0 : } else if (strcmp(arg, "--hs-debug") == 0) {
295 0 : hs_debug_flag = atoi(argv[iarg+1]);
296 0 : iarg++;
297 0 : } else if (strcmp(arg, "--odbc") == 0) {
298 0 : mh = MakeMidasHistoryODBC();
299 0 : assert(mh);
300 0 : mh->hs_set_debug(hs_debug_flag);
301 0 : status = mh->hs_connect(argv[iarg+1]);
302 0 : if (status != HS_SUCCESS)
303 0 : exit(1);
304 0 : iarg++;
305 0 : } else if (strcmp(arg, "--sqlite") == 0) {
306 0 : mh = MakeMidasHistorySqlite();
307 0 : assert(mh);
308 0 : mh->hs_set_debug(hs_debug_flag);
309 0 : status = mh->hs_connect(argv[iarg+1]);
310 0 : if (status != HS_SUCCESS)
311 0 : exit(1);
312 0 : iarg++;
313 0 : } else if (strcmp(arg, "--mysql") == 0) {
314 0 : mh = MakeMidasHistoryMysql();
315 0 : assert(mh);
316 0 : mh->hs_set_debug(hs_debug_flag);
317 0 : status = mh->hs_connect(argv[iarg+1]);
318 0 : if (status != HS_SUCCESS)
319 0 : exit(1);
320 0 : iarg++;
321 0 : } else if (strcmp(arg, "--pgsql") == 0) {
322 0 : mh = MakeMidasHistoryPgsql();
323 0 : assert(mh);
324 0 : mh->hs_set_debug(hs_debug_flag);
325 0 : status = mh->hs_connect(argv[iarg+1]);
326 0 : if (status != HS_SUCCESS)
327 0 : exit(1);
328 0 : iarg++;
329 0 : } else if (strcmp(arg, "--file") == 0) {
330 0 : mh = MakeMidasHistoryFile();
331 0 : assert(mh);
332 0 : mh->hs_set_debug(hs_debug_flag);
333 0 : status = mh->hs_connect(argv[iarg+1]);
334 0 : if (status != HS_SUCCESS)
335 0 : exit(1);
336 0 : iarg++;
337 : }
338 : }
339 :
340 0 : if (!mh) {
341 0 : status = hs_get_history(hDB, 0, HS_GET_DEFAULT|HS_GET_WRITER|HS_GET_INACTIVE, hs_debug_flag, &mh);
342 0 : assert(status == HS_SUCCESS);
343 0 : assert(mh);
344 : }
345 :
346 0 : for (int iarg=1; iarg<argc; iarg++) {
347 0 : const char* arg = argv[iarg];
348 :
349 0 : if (arg[0] != '-')
350 0 : continue;
351 :
352 0 : if (strcmp(arg, "--hs-debug") == 0) {
353 0 : mh->hs_set_debug(atoi(argv[iarg+1]));
354 0 : iarg++;
355 : }
356 : }
357 :
358 0 : for (int iarg=1; iarg<argc; iarg++) {
359 0 : const char* arg = argv[iarg];
360 :
361 0 : if (strstr(arg, ".hst") != NULL) {
362 0 : copyHst(arg, mh);
363 : }
364 : }
365 :
366 0 : mh->hs_disconnect();
367 :
368 0 : cm_disconnect_experiment();
369 :
370 0 : return 0;
371 : }
372 :
373 : /* emacs
374 : * Local Variables:
375 : * tab-width: 8
376 : * c-basic-offset: 3
377 : * indent-tabs-mode: nil
378 : * End:
379 : */
|
