summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMarko Kreen2012-11-02 09:56:56 +0000
committerMarko Kreen2012-11-02 09:56:56 +0000
commitab4ac0cd39b2a75eaabeb43dafa5f4ee3757500c (patch)
tree2fb3d5989641f3ba2ea3075ddc61952601c8b0eb
parent858cb7c48a67afcb4603f4f3a710e278b88e7a52 (diff)
parentfddcbe19e1623e585f85610cecff50c00f26d228 (diff)
Merge remote-tracking branch 'intgit/master'
-rw-r--r--python/pgq/consumer.py26
-rw-r--r--python/pgq/localconsumer.py9
-rwxr-xr-xpython/skytools/dbservice.py31
-rw-r--r--python/skytools/scripting.py18
-rwxr-xr-xpython/walmgr.py4
5 files changed, 48 insertions, 40 deletions
diff --git a/python/pgq/consumer.py b/python/pgq/consumer.py
index b3b45d18..aa60ffa8 100644
--- a/python/pgq/consumer.py
+++ b/python/pgq/consumer.py
@@ -15,9 +15,9 @@ __all__ = ['Consumer']
class _WalkerEvent(Event):
"""Redirects status flags to BatchWalker.
-
- That way event data can gc-d immidiately and
- tag_done() events dont need to be remembered.
+
+ That way event data can be gc'd immediately and
+ tag_done() events don't need to be remembered.
"""
def __init__(self, walker, queue, row):
Event.__init__(self, queue, row)
@@ -122,8 +122,8 @@ class Consumer(skytools.DBScript):
# the actual user script on top of pgq.Consumer must also support it
#pgq_autocommit = 0
- # whether to wait for specified number of events, before
- # assigning a batch (0 disables)
+ # whether to wait for specified number of events,
+ # before assigning a batch (0 disables)
#pgq_batch_collect_events = 0
# whether to wait specified amount of time,
@@ -131,7 +131,7 @@ class Consumer(skytools.DBScript):
#pgq_batch_collect_interval =
# whether to stay behind queue top (postgres interval)
- #pgq_keep_lag =
+ #pgq_keep_lag =
# in how many seconds to write keepalive stats for idle consumers
# this stats is used for detecting that consumer is still running
@@ -167,7 +167,7 @@ class Consumer(skytools.DBScript):
def __init__(self, service_name, db_name, args):
"""Initialize new consumer.
-
+
@param service_name: service_name for DBScript
@param db_name: name of database for get_database()
@param args: cmdline args for DBScript
@@ -242,15 +242,15 @@ class Consumer(skytools.DBScript):
def process_event(self, db, event):
"""Process one event.
- Should be overrided by user code.
+ Should be overridden by user code.
"""
raise Exception("needs to be implemented")
def process_batch(self, db, batch_id, event_list):
"""Process all events in batch.
-
+
By default calls process_event for each.
- Can be overrided by user code.
+ Can be overridden by user code.
"""
for ev in event_list:
self.process_event(db, ev)
@@ -275,7 +275,7 @@ class Consumer(skytools.DBScript):
# load events
ev_list = self._load_batch_events(curs, batch_id)
db.commit()
-
+
# process events
self._launch_process_batch(db, batch_id, ev_list)
@@ -302,7 +302,7 @@ class Consumer(skytools.DBScript):
db = self.get_database(self.db_name)
cx = db.cursor()
cx.execute("select pgq.unregister_consumer(%s, %s)",
- [self.queue_name, self.consumer_name])
+ [self.queue_name, self.consumer_name])
db.commit()
def _launch_process_batch(self, db, batch_id, list):
@@ -398,4 +398,4 @@ class Consumer(skytools.DBScript):
self.stat_put('duration', round(t - self.stat_batch_start,4))
if count > 0: # reset timer if we got some events
self.stat_put('idle', round(self.stat_batch_start - self.idle_start,4))
- self.idle_start = t \ No newline at end of file
+ self.idle_start = t
diff --git a/python/pgq/localconsumer.py b/python/pgq/localconsumer.py
index 8cee2cc3..5ba74453 100644
--- a/python/pgq/localconsumer.py
+++ b/python/pgq/localconsumer.py
@@ -26,7 +26,7 @@ class LocalConsumer(pgq.Consumer):
Features:
- Can detect if several batches are already applied to dest db.
- - If some ticks are lost. allows to seek back on queue.
+ - If some ticks are lost, allows to seek back on queue.
Whether it succeeds, depends on pgq configuration.
Config options::
@@ -84,7 +84,7 @@ class LocalConsumer(pgq.Consumer):
q = "select * from pgq.register_consumer(%s, %s)"
curs.execute(q, [self.queue_name, self.consumer_name])
elif local_tick < 0:
- self.log.info("Local tick missing, storing queueu tick %d", queue_tick)
+ self.log.info("Local tick missing, storing queue tick %d", queue_tick)
self.save_local_tick(queue_tick)
elif local_tick > queue_tick:
self.log.warning("Tracking out of sync: queue=%d local=%d. Repositioning on queue. [Database failure?]",
@@ -177,14 +177,14 @@ class LocalConsumer(pgq.Consumer):
src_db = self.get_database(self.db_name)
src_curs = src_db.cursor()
- self.log.info("Rewinding queue to tick local tick %d", dst_tick)
+ self.log.info("Rewinding queue to local tick %d", dst_tick)
q = "select pgq.register_consumer_at(%s, %s, %s)"
src_curs.execute(q, [self.queue_name, self.consumer_name, dst_tick])
src_db.commit()
else:
self.log.error('Cannot rewind, no tick found in local file')
-
+
def dst_reset(self):
self.log.info("Removing local tracking file")
try:
@@ -213,4 +213,3 @@ class LocalConsumer(pgq.Consumer):
"""Store tick in local file."""
data = str(tick_id)
skytools.write_atomic(self.local_tracking_file, data)
-
diff --git a/python/skytools/dbservice.py b/python/skytools/dbservice.py
index d1ddb666..7ca5956c 100755
--- a/python/skytools/dbservice.py
+++ b/python/skytools/dbservice.py
@@ -22,7 +22,7 @@ def transform_fields(rows, key_fields, name_field, data_field):
"""Convert multiple-rows per key input array
to one-row, multiple-column output array. The input arrays
must be sorted by the key fields.
-
+
>>> rows = []
>>> rows.append({'time': '22:00', 'metric': 'count', 'value': 100})
>>> rows.append({'time': '22:00', 'metric': 'dur', 'value': 7})
@@ -152,21 +152,21 @@ def log_result(log, list):
class DBService:
- """ Wrap parametrisized query handling and multiset stored procedure writing
+ """ Wrap parameterized query handling and multiset stored procedure writing
"""
ROW = "_row" # name of the fake field where internal record id is stored
FIELD = "_field" # parameter name for the field in record that is related to current message
PARAM = "_param" # name of the parameter to which message relates
SKIP = "skip" # used when record is needed for it's data but is not been updated
- INSERT = "insert"
+ INSERT = "insert"
UPDATE = "update"
DELETE = "delete"
- INFO = "info" # just informative message for the user
+ INFO = "info" # just informative message for the user
NOTICE = "notice" # more than info less than warning
WARNING = "warning" # warning message, something is out of ordinary
ERROR = "error" # error found but execution continues until check then error is raised
FATAL = "fatal" # execution is terminated at once and all found errors returned
-
+
def __init__(self, context, global_dict = None):
""" This object must be initiated in the beginning of each db service
"""
@@ -175,19 +175,19 @@ class DBService:
self.global_dict = global_dict # used for cacheing query plans
self._retval = [] # used to collect return resultsets
self._is_test = 'is_test' in rec # used to convert output into human readable form
-
- self.sqls = None # if sqls stays None then no recording of sqls is done
+
+ self.sqls = None # if sqls stays None then no recording of sqls is done
if "show_sql" in rec: # api must add exected sql to resultset
self.sqls = [] # sql's executed by dbservice, used for dubugging
-
+
self.can_save = True # used to keep value most severe error found so far
self.messages = [] # used to hold list of messages to be returned to the user
-
+
# error and message handling
-
+
def tell_user(self, severity, code, message, params = None, **kvargs):
""" Adds another message to the set of messages to be sent back to user
- If error message then can_save is set false
+ If error message then can_save is set false
If fatal message then error or found errors are raised at once
"""
params = params or kvargs
@@ -210,14 +210,14 @@ class DBService:
msgs = "Dbservice error(s): " + make_record_array( self.messages )
plpy.error( msgs )
- # run sql meant mostly for select but not limited to
+ # run sql meant mostly for select but not limited to
def create_query(self, sql, params = None, **kvargs):
""" Returns initialized querybuilder object for building complex dynamic queries
"""
params = params or kvargs
return skytools.PLPyQueryBuilder(sql, params, self.global_dict, self.sqls )
-
+
def run_query(self, sql, params = None, **kvargs):
""" Helper function if everything you need is just paramertisized execute
Sets rows_found that is coneninet to use when you don't need result just
@@ -265,7 +265,7 @@ class DBService:
return row.values()[0]
# resultset handling
-
+
def return_next(self, rows, res_name, severity = None):
""" Adds given set of rows to resultset
"""
@@ -273,7 +273,7 @@ class DBService:
if severity is not None and len(rows) == 0:
self.tell_user(severity, "dbsXXXX", "No matching records found")
return rows
-
+
def return_next_sql(self, sql, params, res_name, severity = None):
""" Exectes query and adds recors resultset
"""
@@ -587,4 +587,3 @@ class ServiceContext(DBService):
If dict was not provied with call it is created
"""
return fields
-
diff --git a/python/skytools/scripting.py b/python/skytools/scripting.py
index ff8eac79..38e1aee4 100644
--- a/python/skytools/scripting.py
+++ b/python/skytools/scripting.py
@@ -249,7 +249,7 @@ class BaseScript(object):
@param service_name: unique name for script.
It will be also default job_name, if not specified in config.
- @param args: cmdline args (sys.argv[1:]), but can be overrided
+ @param args: cmdline args (sys.argv[1:]), but can be overridden
"""
self.service_name = service_name
self.go_daemon = 0
@@ -361,7 +361,7 @@ class BaseScript(object):
"""Loads and returns skytools.Config instance.
By default it uses first command-line argument as config
- file name. Can be overrided.
+ file name. Can be overridden.
"""
if len(self.args) < 1:
@@ -376,7 +376,7 @@ class BaseScript(object):
"""Initialize a OptionParser() instance that will be used to
parse command line arguments.
- Note that it can be overrided both directions - either DBScript
+ Note that it can be overridden both directions - either DBScript
will initialize a instance and passes to user code or user can
initialize and then pass to DBScript.init_optparse().
@@ -478,6 +478,14 @@ class BaseScript(object):
sys.exit(1)
self.last_sigint = t
+ def stat_get(self, key):
+ """Reads a stat value."""
+ try:
+ value = self.stat_dict[key]
+ except KeyError:
+ value = None
+ return value
+
def stat_put(self, key, value):
"""Sets a stat value."""
self.stat_dict[key] = value
@@ -619,6 +627,7 @@ class BaseScript(object):
In case of daemon, if will be called in same process as work(),
unlike __init__().
"""
+ self.started = time.time()
# set signals
if hasattr(signal, 'SIGHUP'):
@@ -669,7 +678,7 @@ class DBScript(BaseScript):
@param service_name: unique name for script.
It will be also default job_name, if not specified in config.
- @param args: cmdline args (sys.argv[1:]), but can be overrided
+ @param args: cmdline args (sys.argv[1:]), but can be overridden
"""
self.db_cache = {}
self._db_defaults = {}
@@ -894,7 +903,6 @@ class DBScript(BaseScript):
# error is already logged
sys.exit(1)
-
def listen(self, dbname, channel):
"""Make connection listen for specific event channel.
diff --git a/python/walmgr.py b/python/walmgr.py
index 408d8650..edbfaf64 100755
--- a/python/walmgr.py
+++ b/python/walmgr.py
@@ -1502,7 +1502,6 @@ STOP TIME: %(stop_time)s
self.log.debug("%s: start copy", srcname)
self.master_periodic()
- self.set_last_complete(srcname)
dst_loc = self.cf.getfile("completed_wals")
if dst_loc[-1] != "/":
@@ -1517,6 +1516,9 @@ STOP TIME: %(stop_time)s
cmdline = ["ssh", "-nT", slave, "sync" ]
self.exec_cmd(cmdline)
+ # slave has the file now, set markers
+ self.set_last_complete(srcname)
+
self.log.debug("%s: done", srcname)
end_time = time.time()
self.stat_add('count', 1)