> my vote is to bump the ODB size limit to 1999*1000*1000 (not quite 2GB). but this needs to be tested. especially save and restore from ODB, XML and JSON files, including how long it takes to save and load a 1.9GB ODB. K.O.

I had some fun with python and created a test script which can be executed in the MIDASSYS/online folder (test_odb.py). I did not really normalize the time so it will be different at different systems but I guess the trend is important (see create_time.pdf).
What is surprising to me is that even that I only write one STRING key to the time increases. Is this maybe related to what Stefan said about the run start - so that odbedit needs some time to load the bigger ODB?
Second thing is that also the creation / storing and load time is increasing. Should this be or is there a bug in the code I use or again is this related to the previous point?

The test of comparing the ODB after store / load / store already fails for the json format. I know I only test if the dicts are the same, so for timestamps this already fails.
But what is strange here is that sometimes the test works sometimes not and its different from run to run.

I will try to improve the test a bit more but for a short update this is how it looks so fare.

Best,
Marius

import os, time, json, random, string
import matplotlib.pyplot as plt
from matplotlib import colors
import numpy as np


def read_json(path):
    with open(path, "r") as f:
        return json.load(f)

def clean_odb():
    cmd = "rm .*.SHM >/dev/null"
    os.system(cmd)
    cmd = "rm .SHM_TYPE.TXT >/dev/null"
    os.system(cmd)
    cmd = "rm .SHM_HOST.TXT >/dev/null"
    os.system(cmd)
    cmd = "rm /dev/shm/*_SHM >/dev/null"
    os.system(cmd)
    cmd = "rm /dev/shm/*online_online* >/dev/null"
    os.system(cmd)
    cmd = "rm .ODB_SIZE.TXT >/dev/null"
    os.system(cmd)

def create_odb(arg):
    size = arg["size"]
    cmd = f"odbinit -s {size}MB >/dev/null"
    os.system(cmd)

def time_function(func, **args):
    start = time.time()
    if not args:
        func()
    if args:
        func(args)
    return time.time() - start

def randomword(length=10):
   letters = string.ascii_lowercase
   return ''.join(random.choice(letters) for i in range(length))

def fill_odb(arg):
    size = arg["size"]
    num_key_values = arg["num_key_values"]
    cmd = "odbedit -c \'mkdir Test\' >/dev/null"
    os.system(cmd)
    # one string key has 32 bytes for data and 72 bytes for key
    # one INT32 key has 38 bytes for data and 72  bytes for key
    # 1MB is 8388608 bits so 524288 bytes ODB per key / data
    rounds = int(size * 524288 / 72)
    for idx in range(num_key_values):
        word = randomword()
        cmd = f"odbedit -d Test -c \'create STRING {word}\' >/dev/null"
        os.system(cmd)
        cmd = f"odbedit -c \'set Test/{word} {word}\' >/dev/null"
        os.system(cmd)

def store_odb():
    cmd = "odbedit -c \'save test.json\' >/dev/null"
    os.system(cmd)

def load_odb():
    cmd = "odbedit -c \'load test.json\' >/dev/null"
    os.system(cmd)

def check_dict(dictA, dictB, value):
    if dictA[value] == dictB[value]:
        return 0
    return 1

create_time = []
fill_time = []
store_time = []
load_time = []
time_random = time_function(randomword)
MBs = 100#1500
num_key_values = 1
key_list = ["/MIDAS version", "/MIDAS git revision", "/ODB path", "/filename", "Experiment", "System", "Programs", "Logger", "Test", "Runinfo", "Alarms"]
fails = np.zeros([len(range(1, MBs, 10)), len(key_list)])
for i, size in enumerate(range(1, MBs, 10)):
    clean_odb()
    create_time.append(time_function(create_odb, size=size))
    cur_fill_time = time_function(fill_odb, size=size, num_key_values=num_key_values)
    fill_time.append(cur_fill_time - time_random * num_key_values)
    cmd = "odbedit -c mem"
    os.system(cmd)
    print(f"Test size {size}MB")
    store_time.append(time_function(store_odb))
    dictA = read_json("test.json")
    load_time.append(time_function(load_odb))
    store_odb()
    dictB = read_json("test.json")
    for j, k in enumerate(key_list):
        fails[i][j] = check_dict(dictA, dictB, k)

plt.plot(range(1, MBs, 10), create_time, ".", label="create")
plt.plot(range(1, MBs, 10), fill_time, ".", label=f"fill {num_key_values} key")
plt.plot(range(1, MBs, 10), store_time, ".", label="store")
plt.plot(range(1, MBs, 10), load_time, ".", label="load")
plt.xlabel("size in MB")
plt.ylabel("time in sec")
plt.legend()
plt.savefig("create_time.pdf")
plt.close()

fig, ax = plt.subplots()
cmap = colors.ListedColormap(['green','red'])
plt.pcolor(fails[::-1],cmap=cmap, edgecolors='k', linewidths=3)
ax.set_xticks(range(len(key_list)))
ax.set_xticklabels(key_list, rotation=45, horizontalalignment='right')
ax.set_yticks(range(len(range(1, MBs, 10))))
ax.set_yticklabels([str(i)+"MB" for i in range(1, MBs, 10)])
plt.savefig("fails.pdf", bbox_inches="tight")