diff options
author | Marko Kreen | 2012-11-02 09:56:56 +0000 |
---|---|---|
committer | Marko Kreen | 2012-11-02 09:56:56 +0000 |
commit | ab4ac0cd39b2a75eaabeb43dafa5f4ee3757500c (patch) | |
tree | 2fb3d5989641f3ba2ea3075ddc61952601c8b0eb | |
parent | 858cb7c48a67afcb4603f4f3a710e278b88e7a52 (diff) | |
parent | fddcbe19e1623e585f85610cecff50c00f26d228 (diff) |
Merge remote-tracking branch 'intgit/master'
-rw-r--r-- | python/pgq/consumer.py | 26 | ||||
-rw-r--r-- | python/pgq/localconsumer.py | 9 | ||||
-rwxr-xr-x | python/skytools/dbservice.py | 31 | ||||
-rw-r--r-- | python/skytools/scripting.py | 18 | ||||
-rwxr-xr-x | python/walmgr.py | 4 |
5 files changed, 48 insertions, 40 deletions
diff --git a/python/pgq/consumer.py b/python/pgq/consumer.py index b3b45d18..aa60ffa8 100644 --- a/python/pgq/consumer.py +++ b/python/pgq/consumer.py @@ -15,9 +15,9 @@ __all__ = ['Consumer'] class _WalkerEvent(Event): """Redirects status flags to BatchWalker. - - That way event data can gc-d immidiately and - tag_done() events dont need to be remembered. + + That way event data can be gc'd immediately and + tag_done() events don't need to be remembered. """ def __init__(self, walker, queue, row): Event.__init__(self, queue, row) @@ -122,8 +122,8 @@ class Consumer(skytools.DBScript): # the actual user script on top of pgq.Consumer must also support it #pgq_autocommit = 0 - # whether to wait for specified number of events, before - # assigning a batch (0 disables) + # whether to wait for specified number of events, + # before assigning a batch (0 disables) #pgq_batch_collect_events = 0 # whether to wait specified amount of time, @@ -131,7 +131,7 @@ class Consumer(skytools.DBScript): #pgq_batch_collect_interval = # whether to stay behind queue top (postgres interval) - #pgq_keep_lag = + #pgq_keep_lag = # in how many seconds to write keepalive stats for idle consumers # this stats is used for detecting that consumer is still running @@ -167,7 +167,7 @@ class Consumer(skytools.DBScript): def __init__(self, service_name, db_name, args): """Initialize new consumer. - + @param service_name: service_name for DBScript @param db_name: name of database for get_database() @param args: cmdline args for DBScript @@ -242,15 +242,15 @@ class Consumer(skytools.DBScript): def process_event(self, db, event): """Process one event. - Should be overrided by user code. + Should be overridden by user code. """ raise Exception("needs to be implemented") def process_batch(self, db, batch_id, event_list): """Process all events in batch. - + By default calls process_event for each. - Can be overrided by user code. + Can be overridden by user code. """ for ev in event_list: self.process_event(db, ev) @@ -275,7 +275,7 @@ class Consumer(skytools.DBScript): # load events ev_list = self._load_batch_events(curs, batch_id) db.commit() - + # process events self._launch_process_batch(db, batch_id, ev_list) @@ -302,7 +302,7 @@ class Consumer(skytools.DBScript): db = self.get_database(self.db_name) cx = db.cursor() cx.execute("select pgq.unregister_consumer(%s, %s)", - [self.queue_name, self.consumer_name]) + [self.queue_name, self.consumer_name]) db.commit() def _launch_process_batch(self, db, batch_id, list): @@ -398,4 +398,4 @@ class Consumer(skytools.DBScript): self.stat_put('duration', round(t - self.stat_batch_start,4)) if count > 0: # reset timer if we got some events self.stat_put('idle', round(self.stat_batch_start - self.idle_start,4)) - self.idle_start = t
\ No newline at end of file + self.idle_start = t diff --git a/python/pgq/localconsumer.py b/python/pgq/localconsumer.py index 8cee2cc3..5ba74453 100644 --- a/python/pgq/localconsumer.py +++ b/python/pgq/localconsumer.py @@ -26,7 +26,7 @@ class LocalConsumer(pgq.Consumer): Features: - Can detect if several batches are already applied to dest db. - - If some ticks are lost. allows to seek back on queue. + - If some ticks are lost, allows to seek back on queue. Whether it succeeds, depends on pgq configuration. Config options:: @@ -84,7 +84,7 @@ class LocalConsumer(pgq.Consumer): q = "select * from pgq.register_consumer(%s, %s)" curs.execute(q, [self.queue_name, self.consumer_name]) elif local_tick < 0: - self.log.info("Local tick missing, storing queueu tick %d", queue_tick) + self.log.info("Local tick missing, storing queue tick %d", queue_tick) self.save_local_tick(queue_tick) elif local_tick > queue_tick: self.log.warning("Tracking out of sync: queue=%d local=%d. Repositioning on queue. [Database failure?]", @@ -177,14 +177,14 @@ class LocalConsumer(pgq.Consumer): src_db = self.get_database(self.db_name) src_curs = src_db.cursor() - self.log.info("Rewinding queue to tick local tick %d", dst_tick) + self.log.info("Rewinding queue to local tick %d", dst_tick) q = "select pgq.register_consumer_at(%s, %s, %s)" src_curs.execute(q, [self.queue_name, self.consumer_name, dst_tick]) src_db.commit() else: self.log.error('Cannot rewind, no tick found in local file') - + def dst_reset(self): self.log.info("Removing local tracking file") try: @@ -213,4 +213,3 @@ class LocalConsumer(pgq.Consumer): """Store tick in local file.""" data = str(tick_id) skytools.write_atomic(self.local_tracking_file, data) - diff --git a/python/skytools/dbservice.py b/python/skytools/dbservice.py index d1ddb666..7ca5956c 100755 --- a/python/skytools/dbservice.py +++ b/python/skytools/dbservice.py @@ -22,7 +22,7 @@ def transform_fields(rows, key_fields, name_field, data_field): """Convert multiple-rows per key input array to one-row, multiple-column output array. The input arrays must be sorted by the key fields. - + >>> rows = [] >>> rows.append({'time': '22:00', 'metric': 'count', 'value': 100}) >>> rows.append({'time': '22:00', 'metric': 'dur', 'value': 7}) @@ -152,21 +152,21 @@ def log_result(log, list): class DBService: - """ Wrap parametrisized query handling and multiset stored procedure writing + """ Wrap parameterized query handling and multiset stored procedure writing """ ROW = "_row" # name of the fake field where internal record id is stored FIELD = "_field" # parameter name for the field in record that is related to current message PARAM = "_param" # name of the parameter to which message relates SKIP = "skip" # used when record is needed for it's data but is not been updated - INSERT = "insert" + INSERT = "insert" UPDATE = "update" DELETE = "delete" - INFO = "info" # just informative message for the user + INFO = "info" # just informative message for the user NOTICE = "notice" # more than info less than warning WARNING = "warning" # warning message, something is out of ordinary ERROR = "error" # error found but execution continues until check then error is raised FATAL = "fatal" # execution is terminated at once and all found errors returned - + def __init__(self, context, global_dict = None): """ This object must be initiated in the beginning of each db service """ @@ -175,19 +175,19 @@ class DBService: self.global_dict = global_dict # used for cacheing query plans self._retval = [] # used to collect return resultsets self._is_test = 'is_test' in rec # used to convert output into human readable form - - self.sqls = None # if sqls stays None then no recording of sqls is done + + self.sqls = None # if sqls stays None then no recording of sqls is done if "show_sql" in rec: # api must add exected sql to resultset self.sqls = [] # sql's executed by dbservice, used for dubugging - + self.can_save = True # used to keep value most severe error found so far self.messages = [] # used to hold list of messages to be returned to the user - + # error and message handling - + def tell_user(self, severity, code, message, params = None, **kvargs): """ Adds another message to the set of messages to be sent back to user - If error message then can_save is set false + If error message then can_save is set false If fatal message then error or found errors are raised at once """ params = params or kvargs @@ -210,14 +210,14 @@ class DBService: msgs = "Dbservice error(s): " + make_record_array( self.messages ) plpy.error( msgs ) - # run sql meant mostly for select but not limited to + # run sql meant mostly for select but not limited to def create_query(self, sql, params = None, **kvargs): """ Returns initialized querybuilder object for building complex dynamic queries """ params = params or kvargs return skytools.PLPyQueryBuilder(sql, params, self.global_dict, self.sqls ) - + def run_query(self, sql, params = None, **kvargs): """ Helper function if everything you need is just paramertisized execute Sets rows_found that is coneninet to use when you don't need result just @@ -265,7 +265,7 @@ class DBService: return row.values()[0] # resultset handling - + def return_next(self, rows, res_name, severity = None): """ Adds given set of rows to resultset """ @@ -273,7 +273,7 @@ class DBService: if severity is not None and len(rows) == 0: self.tell_user(severity, "dbsXXXX", "No matching records found") return rows - + def return_next_sql(self, sql, params, res_name, severity = None): """ Exectes query and adds recors resultset """ @@ -587,4 +587,3 @@ class ServiceContext(DBService): If dict was not provied with call it is created """ return fields - diff --git a/python/skytools/scripting.py b/python/skytools/scripting.py index ff8eac79..38e1aee4 100644 --- a/python/skytools/scripting.py +++ b/python/skytools/scripting.py @@ -249,7 +249,7 @@ class BaseScript(object): @param service_name: unique name for script. It will be also default job_name, if not specified in config. - @param args: cmdline args (sys.argv[1:]), but can be overrided + @param args: cmdline args (sys.argv[1:]), but can be overridden """ self.service_name = service_name self.go_daemon = 0 @@ -361,7 +361,7 @@ class BaseScript(object): """Loads and returns skytools.Config instance. By default it uses first command-line argument as config - file name. Can be overrided. + file name. Can be overridden. """ if len(self.args) < 1: @@ -376,7 +376,7 @@ class BaseScript(object): """Initialize a OptionParser() instance that will be used to parse command line arguments. - Note that it can be overrided both directions - either DBScript + Note that it can be overridden both directions - either DBScript will initialize a instance and passes to user code or user can initialize and then pass to DBScript.init_optparse(). @@ -478,6 +478,14 @@ class BaseScript(object): sys.exit(1) self.last_sigint = t + def stat_get(self, key): + """Reads a stat value.""" + try: + value = self.stat_dict[key] + except KeyError: + value = None + return value + def stat_put(self, key, value): """Sets a stat value.""" self.stat_dict[key] = value @@ -619,6 +627,7 @@ class BaseScript(object): In case of daemon, if will be called in same process as work(), unlike __init__(). """ + self.started = time.time() # set signals if hasattr(signal, 'SIGHUP'): @@ -669,7 +678,7 @@ class DBScript(BaseScript): @param service_name: unique name for script. It will be also default job_name, if not specified in config. - @param args: cmdline args (sys.argv[1:]), but can be overrided + @param args: cmdline args (sys.argv[1:]), but can be overridden """ self.db_cache = {} self._db_defaults = {} @@ -894,7 +903,6 @@ class DBScript(BaseScript): # error is already logged sys.exit(1) - def listen(self, dbname, channel): """Make connection listen for specific event channel. diff --git a/python/walmgr.py b/python/walmgr.py index 408d8650..edbfaf64 100755 --- a/python/walmgr.py +++ b/python/walmgr.py @@ -1502,7 +1502,6 @@ STOP TIME: %(stop_time)s self.log.debug("%s: start copy", srcname) self.master_periodic() - self.set_last_complete(srcname) dst_loc = self.cf.getfile("completed_wals") if dst_loc[-1] != "/": @@ -1517,6 +1516,9 @@ STOP TIME: %(stop_time)s cmdline = ["ssh", "-nT", slave, "sync" ] self.exec_cmd(cmdline) + # slave has the file now, set markers + self.set_last_complete(srcname) + self.log.debug("%s: done", srcname) end_time = time.time() self.stat_add('count', 1) |