Public Member Functions | |
| def | __init__ |
| def | AddRow |
| def | Apply |
| def | CanApply |
| def | RemoveSeqno |
Public Attributes | |
| parent | |
| debug | |
| dbi | |
| is_global | |
| failed | |
| applied | |
| num_conv_unsign | |
| rows | |
| table_name | |
| start_date | |
| end_date | |
| detectormask | |
| simmask | |
| epoch | |
| aggregate | |
| task | |
| creation_date | |
| seqno | |
A table update for a single aggregate.
Definition at line 196 of file database_updater.py.
| def database_updater::TableUpdate::__init__ | ( | self, | ||
| parent, | ||||
| begin_line, | ||||
| is_global | ||||
| ) |
Definition at line 199 of file database_updater.py.
00199 : 00200 self.parent = parent # Parent DatabaseUpdater 00201 self.debug = parent.debug # Debug flag 00202 self.dbi = parent.dbi # Database Interface 00203 self.is_global = is_global # Determine local/global SEQNO 00204 self.failed = False # Set True if any error found while assembling update 00205 self.applied = False # Set True once applied. 00206 self.num_conv_unsign = 0 # Number of unsigned integers converted 00207 self.rows = [] # List of rows of data. 00208 self.table_name = "" # Table name parsed from BEGIN_TABLE line 00209 self.start_date = "" # Start date parsed from BEGIN_TABLE line 00210 self.end_date = "" # End date parsed from BEGIN_TABLE line 00211 self.detectormask = "1" # Hardwired, at least for now 00212 self.simmask = "1" # Overridden by the option SIMMASK= 00213 self.epoch = "" # Overridden by the option EPOCH= 00214 self.aggregate = "" # Aggregate number parsed from BEGIN_TABLE line 00215 self.task = "0" # Task number optionally parsed from BEGIN_TABLE line (default 0). 00216 self.creation_date = "" # Creation date parsed from BEGIN_TABLE line 00217 self.seqno = "" # Empty until Apply method executed. 00218 00219 # Parse BEGIN_TABLE line 00220 mo = re.search(r"^BEGIN_TABLE\s+(\w+)\s+'(.*?)'\s+'(.*?)'\s+(\d+)\s+'(.*?)'(|\s+(.*))$",begin_line) 00221 # BEGIN_TABLE TNAME 'START' ' END ' AGGRE 'CREAT' dummy 00222 # dummy is matched to <space><options> if 1 or more options exist otherwise is the empty string 00223 # if dummy != "" then <options> is matched to options 00224 if not mo: 00225 print "Failing update; cannot parse line: " + begin_line 00226 self.failed = True 00227 return 00228 (self.table_name,self.start_date,self.end_date,self.aggregate,self.creation_date,dummy,options) = mo.groups() 00229 if options: 00230 00231 # Parse out {<task>} {key=value key=value ...} 00232 options = options.lower() 00233 #next two lines grab the task number if it is supplied as a lone digit 00234 #(can be over-written by a subsequent TASK=N) 00235 mo = re.match(r"\s*(\d+)(|\s+(.*))$",options) 00236 if mo: (self.task,dummy,options) = mo.groups() 00237 while options: 00238 mo = re.search(r"^(\S+)=(\S+)(|\s+(.*))$",options) 00239 if not mo: 00240 print "Failing update; cannot parse options: '%s'" % options 00241 self.failed = True 00242 return 00243 (key,value,dummy,options) = mo.groups() 00244 00245 # Deal with TASK= 00246 if key == "task": 00247 print "TASK from key/value" 00248 self.task = int(value) 00249 00250 # Deal with SIMMASK= 00251 elif key == "simmask": 00252 if value == "data" : self.simmask = "1" 00253 elif value == "mc" : self.simmask = "4" 00254 elif value == "all" : self.simmask = "-1" 00255 else: 00256 print "Failing update; bad SIMMASK option value: '%s'" % value 00257 self.failed = True 00258 return 00259 00260 # Deal with EPOCH= 00261 elif key == "epoch": 00262 try: 00263 val_int = int(value) 00264 if val_int >= 0 and val_int <= 100: self.epoch = value 00265 else: raise 00266 except: 00267 print "Failing update; bad EPOCH option value: '%s'" % value 00268 self.failed = True 00269 return 00270 00271 else: 00272 print "Failing update; bad option key: '%s'" % key 00273 self.failed = True 00274 return 00275 00276 if not self.parent.dbi.TableExists(self.table_name): 00277 print "Failing update; table %s does not exist." % self.table_name 00278 self.failed = True 00279 return 00280 for date in (self.start_date,self.end_date,self.creation_date): 00281 if not re.search(r'\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d',date): 00282 print "Failing update; bad date '%s' in %s." % (date,begin_line) 00283 self.failed = True 00284 return 00285 00286 #--------------------------------------------- TableUpdate -------------------------------------------------- 00287 def AddRow(self,row_line):
| def database_updater::TableUpdate::AddRow | ( | self, | ||
| row_line | ||||
| ) |
Add a row.
Definition at line 288 of file database_updater.py.
00288 : 00289 """Add a row.""" 00290 00291 # Look for unsigned integers to convert if required. 00292 if ( not self.parent.convert_unsigned ) : 00293 self.rows.append(row_line.strip()) 00294 return 00295 converted_line = "" 00296 for value in row_line.strip().split(','): 00297 try: 00298 int_value = int(value) 00299 if int_value > 2147483647: 00300 int_value -= 4294967296 00301 self.num_conv_unsign += 1 00302 value = str(int_value) 00303 except: pass 00304 if converted_line: converted_line += "," 00305 converted_line += value 00306 self.rows.append(converted_line) 00307 00308 00309 #--------------------------------------------- TableUpdate -------------------------------------------------- 00310 def Apply(self):
| def database_updater::TableUpdate::Apply | ( | self | ) |
Apply update and return True if successful. The table must already exist but it's VLD will be created if necessary. If it fails it attempts to remove any data already committed.
Definition at line 311 of file database_updater.py.
00311 : 00312 """Apply update and return True if successful. 00313 The table must already exist but it's VLD will be created if 00314 necessary. 00315 If it fails it attempts to remove any data already committed.""" 00316 00317 if not self.CanApply(): return False 00318 00319 # If required, create VLD table. 00320 if not self.dbi.TableExists(self.table_name + "VLD"): 00321 print " Creating table %sVLD" % self.table_name 00322 sql = "CREATE TABLE %sVLD (" % self.table_name 00323 sql += "SEQNO integer not null primary key," 00324 sql += "TIMESTART datetime not null," 00325 sql += "TIMEEND datetime not null," 00326 if self.epoch != "": 00327 sql += "EPOCH tinyint(4)," 00328 sql += "REALITY tinyint(4)," 00329 sql += "DETECTORMASK tinyint(4)," 00330 sql += "SIMMASK tinyint(4)," 00331 sql += "TASK integer," 00332 sql += "AGGREGATENO integer," 00333 sql += "CREATIONDATE datetime not null," 00334 sql += "INSERTDATE datetime not null," 00335 sql += "key TIMESTART (TIMESTART), " 00336 sql += "key TIMEEND (TIMEEND));" 00337 if not self.dbi.Query(sql): return False 00338 00339 print " Applying update to table %s for validity range '%s' - '%s' aggregate %s task %s..." %\ 00340 (self.table_name,self.start_date,self.end_date,self.aggregate,self.task) 00341 if self.num_conv_unsign: print " (%d unsigned integer converted before applying update)" % self.num_conv_unsign 00342 print " requesting sequence number ... ", 00343 require_global = -1 00344 if self.is_global: require_global = 1 00345 cmd = 'allocate_seq_no.exe %s %d | tail -1' % (self.table_name,require_global) 00346 if self.debug: print "About to execute " + cmd 00347 inp = os.popen(cmd,"r") 00348 line = inp.readline() 00349 mo = re.search(r'(\d+)',line) 00350 if mo: self.seqno = int(mo.group(1)) 00351 if not self.seqno: 00352 print "failed to allocate a sequence number!" 00353 self.failed = True 00354 return False 00355 print "allocated sequence number %d" % self.seqno 00356 00357 # Just to be on the safe side remove this sequence number. 00358 self.RemoveSeqno() 00359 00360 # Process all rows adding SEQNO and ROW_COUNTER. 00361 # To improve performance a little group rows together into NUM_ROWS_PER_QUERY to reduce the 00362 # number of MySQL calls. 00363 00364 NUM_ROWS_PER_QUERY = 100 00365 row_num = 0 00366 num_rows = len(self.rows) 00367 sql = "" 00368 00369 while row_num < num_rows: 00370 row = self.rows[row_num] 00371 row_num += 1 00372 if not sql: sql = "INSERT INTO %s VALUES " % self.table_name 00373 sql += "(%d, %d, %s)" % (self.seqno,row_num,row) 00374 if row_num == num_rows or row_num % NUM_ROWS_PER_QUERY == 0: 00375 if not self.dbi.Query(sql): 00376 print "Update has failed, attempting to remove any data." 00377 self.RemoveSeqno() 00378 return False 00379 sql = "" 00380 else: sql += "," 00381 00382 # Add VLD entry 00383 00384 sql = "INSERT INTO %sVLD VALUES (" % self.table_name 00385 sql += "%d, '%s', '%s'," % \ 00386 (self.seqno,self.start_date,self.end_date) 00387 if self.epoch != "": sql += "%s, 0," % self.epoch 00388 sql += "'%s', '%s', %s, %s, '%s', '%s')" % \ 00389 (self.detectormask,self.simmask,self.task,self.aggregate,self.creation_date,\ 00390 datetime.datetime.today().strftime("%Y-%m-%d %H:%M:%S")) 00391 if not self.dbi.Query(sql): 00392 print "Update has failed, attempting to remove any data." 00393 self.RemoveSeqno() 00394 return False 00395 00396 self.applied = True 00397 return True 00398 00399 #--------------------------------------------- TableUpdate -------------------------------------------------- 00400 def CanApply(self):
| def database_updater::TableUpdate::CanApply | ( | self | ) |
Return True if no errors have been found processing the data, it has not already applied and it has some rows.
Definition at line 401 of file database_updater.py.
00401 : 00402 """Return True if no errors have been found processing the data, 00403 it has not already applied and it has some rows.""" 00404 return not self.failed and not self.applied and self.rows 00405 00406 #--------------------------------------------- TableUpdate -------------------------------------------------- 00407 def RemoveSeqno(self):
| def database_updater::TableUpdate::RemoveSeqno | ( | self | ) |
Attempt to remove sequence number.
Definition at line 408 of file database_updater.py.
00408 : 00409 """Attempt to remove sequence number.""" 00410 self.dbi.Query("DELETE FROM %s WHERE SEQNO = %d" % (self.table_name,self.seqno)) 00411 self.dbi.Query("DELETE FROM %sVLD WHERE SEQNO = %d" % (self.table_name,self.seqno)) 00412 00413 #--------------------------------------------- DatabaseUpdater -------------------------------------------------- 00414 #--------------------------------------------- DatabaseUpdater -------------------------------------------------- 00415 #--------------------------------------------- DatabaseUpdater -------------------------------------------------- 00416 class DatabaseUpdater :
Definition at line 214 of file database_updater.py.
Definition at line 205 of file database_updater.py.
Definition at line 216 of file database_updater.py.
Definition at line 202 of file database_updater.py.
Definition at line 201 of file database_updater.py.
Definition at line 211 of file database_updater.py.
Definition at line 210 of file database_updater.py.
Definition at line 213 of file database_updater.py.
Definition at line 204 of file database_updater.py.
Definition at line 203 of file database_updater.py.
Definition at line 206 of file database_updater.py.
Definition at line 200 of file database_updater.py.
Definition at line 207 of file database_updater.py.
Definition at line 217 of file database_updater.py.
Definition at line 212 of file database_updater.py.
Definition at line 209 of file database_updater.py.
Definition at line 208 of file database_updater.py.
Definition at line 215 of file database_updater.py.
1.6.1