pgq: Drop failed event handling.

Badly designed and unused concept.  If such thing is needed
it's much better to handle them via actual queue.

11 files changed, 6 insertions, 360 deletions

diff --git a/doc/skytools3.txt b/doc/skytools3.txt
index 0518dbdf..a539f30e 100644
--- a/doc/skytools3.txt
+++ b/doc/skytools3.txt
@@ -89,6 +89,8 @@ Minor improvements
 
 * psycopg1 is not supported anymore.
 
+* PgQ does not handle "failed events" anymore.
+
 Open questions
 --------------
 
diff --git a/python/pgq/consumer.py b/python/pgq/consumer.py
index 8c001cbe..01f16abd 100644
--- a/python/pgq/consumer.py
+++ b/python/pgq/consumer.py
@@ -27,8 +27,6 @@ class _WalkerEvent(Event):
         self._walker.tag_event_done(self)
     def tag_retry(self, retry_time = 60):
         self._walker.tag_event_retry(self, retry_time)
-    def tag_failed(self, reason):
-        self._walker.tag_failed(self, reason)
     def get_status(self):
         self._walker.get_status(self)
 
@@ -88,9 +86,6 @@ class _BatchWalker(object):
     def tag_event_retry(self, event, retry_time):
         self.status_map[event.id] = (EV_RETRY, retry_time)
 
-    def tag_event_failed(self, event, reason):
-        self.status_map[event.id] = (EV_FAILED, reason)
-
     def get_status(self, event):
         return self.status_map[event.id][0]
 
@@ -153,9 +148,6 @@ class Consumer(skytools.DBScript):
         """Process one event.
 
         Should be overrided by user code.
-
-        Event should be tagged as done, retry or failed.
-        If not, it will be tagged as for retry.
         """
         raise Exception("needs to be implemented")
 
@@ -164,9 +156,6 @@ class Consumer(skytools.DBScript):
         
         By default calls process_event for each.
         Can be overrided by user code.
-
-        Events should be tagged as done, retry or failed.
-        If not, they will be tagged as for retry.
         """
         for ev in event_list:
             self.process_event(db, ev)
@@ -258,23 +247,17 @@ class Consumer(skytools.DBScript):
     def _finish_batch(self, curs, batch_id, list):
         """Tag events and notify that the batch is done."""
 
-        retry = failed = 0
+        retry = 0
         if self.pgq_lazy_fetch:
             for ev_id, stat in list.iter_status():
                 if stat[0] == EV_RETRY:
                     self._tag_retry(curs, batch_id, ev_id, stat[1])
                     retry += 1
-                elif stat[0] == EV_FAILED:
-                    self._tag_failed(curs, batch_id, ev_id, stat[1])
-                    failed += 1
                 elif stat[0] != EV_DONE:
                     raise Exception("Untagged event: id=%d" % ev_id)
         else:
             for ev in list:
-                if ev._status == EV_FAILED:
-                    self._tag_failed(curs, batch_id, ev.id, ev.fail_reason)
-                    failed += 1
-                elif ev._status == EV_RETRY:
+                if ev._status == EV_RETRY:
                     self._tag_retry(curs, batch_id, ev.id, ev.retry_time)
                     retry += 1
                 elif ev._status != EV_DONE:
@@ -284,16 +267,9 @@ class Consumer(skytools.DBScript):
         # report weird events
         if retry:
             self.stat_increase('retry-events', retry)
-        if failed:
-            self.stat_increase('failed-events', failed)
 
         curs.execute("select pgq.finish_batch(%s)", [batch_id])
 
-    def _tag_failed(self, curs, batch_id, ev_id, fail_reason):
-        """Tag event as failed. (internal)"""
-        curs.execute("select pgq.event_failed(%s, %s, %s)",
-                    [batch_id, ev_id, fail_reason])
-
     def _tag_retry(self, cx, batch_id, ev_id, retry_time):
         """Tag event for retry. (internal)"""
         cx.execute("select pgq.event_retry(%s, %s, %s)",
diff --git a/python/pgq/event.py b/python/pgq/event.py
index 80801b16..93745035 100644
--- a/python/pgq/event.py
+++ b/python/pgq/event.py
@@ -2,13 +2,12 @@
 """PgQ event container.
 """
 
-__all__ = ['EV_UNTAGGED', 'EV_RETRY', 'EV_DONE', 'EV_FAILED', 'Event']
+__all__ = ['EV_UNTAGGED', 'EV_RETRY', 'EV_DONE', 'Event']
 
 # Event status codes
 EV_UNTAGGED = -1
 EV_RETRY = 0
 EV_DONE = 1
-EV_FAILED = 2
 
 _fldmap = {
         'ev_id': 'ev_id',
@@ -39,13 +38,12 @@ class Event(object):
     If not, events will stay in retry queue.
     """
     __slots__ = ('_event_row', '_status', 'retry_time',
-                 'fail_reason', 'queue_name')
+                 'queue_name')
 
     def __init__(self, queue_name, row):
         self._event_row = row
         self._status = EV_UNTAGGED
         self.retry_time = 60
-        self.fail_reason = "Buggy consumer"
         self.queue_name = queue_name
 
     def __getattr__(self, key):
@@ -58,10 +56,6 @@ class Event(object):
         self._status = EV_RETRY
         self.retry_time = retry_time
 
-    def tag_failed(self, reason):
-        self._status = EV_FAILED
-        self.fail_reason = reason
-
     def get_status(self):
         return self._status
 
diff --git a/sql/pgq/expected/pgq_core.out b/sql/pgq/expected/pgq_core.out
index a2a83d50..5397a552 100644
--- a/sql/pgq/expected/pgq_core.out
+++ b/sql/pgq/expected/pgq_core.out
@@ -212,23 +212,6 @@ select ev_id,ev_retry,ev_type,ev_data,ev_extra1,ev_extra2,ev_extra3,ev_extra4
 
 close acurs;
 end;
-select * from pgq.failed_event_list('myqueue', 'consumer');
- ev_failed_reason | ev_failed_time | ev_id | ev_time | ev_txid | ev_owner | ev_retry | ev_type | ev_data | ev_extra1 | ev_extra2 | ev_extra3 | ev_extra4 
-------------------+----------------+-------+---------+---------+----------+----------+---------+---------+-----------+-----------+-----------+-----------
-(0 rows)
-
-select pgq.event_failed(3, 1, 'failure test');
- event_failed 
---------------
-            1
-(1 row)
-
-select pgq.event_failed(3, 1, 'failure test');
- event_failed 
---------------
-            0
-(1 row)
-
 select pgq.event_retry(3, 2, 0);
  event_retry 
 -------------
@@ -247,27 +230,6 @@ select pgq.finish_batch(3);
             1
 (1 row)
 
-select ev_failed_reason, ev_id, ev_txid, ev_retry, ev_type, ev_data
-  from pgq.failed_event_list('myqueue', 'consumer');
- ev_failed_reason | ev_id | ev_txid | ev_retry | ev_type | ev_data 
-------------------+-------+---------+----------+---------+---------
- failure test     |     1 |         |        0 | r1      | data
-(1 row)
-
-select ev_failed_reason, ev_id, ev_txid, ev_retry, ev_type, ev_data
-  from pgq.failed_event_list('myqueue', 'consumer', 0, 1);
- ev_failed_reason | ev_id | ev_txid | ev_retry | ev_type | ev_data 
-------------------+-------+---------+----------+---------+---------
-(0 rows)
-
-select * from pgq.failed_event_count('myqueue', 'consumer');
- failed_event_count 
---------------------
-                  1
-(1 row)
-
-select * from pgq.failed_event_delete('myqueue', 'consumer', 0);
-ERROR:  event not found
 select pgq.event_retry_raw('myqueue', 'consumer', now(), 666, now(), 0,
         'rawtest', 'data', null, null, null, null);
  event_retry_raw 
diff --git a/sql/pgq/functions/pgq.event_failed.sql b/sql/pgq/functions/pgq.event_failed.sql
deleted file mode 100644
index 52b5b656..00000000
--- a/sql/pgq/functions/pgq.event_failed.sql
+++ /dev/null
@@ -1,41 +0,0 @@
-create or replace function pgq.event_failed(
-    x_batch_id bigint,
-    x_event_id bigint,
-    x_reason text)
-returns integer as $$
--- ----------------------------------------------------------------------
--- Function: pgq.event_failed(3)
---
---      Copies the event to failed queue so it can be looked at later.
---
--- Parameters:
---      x_batch_id      - ID of active batch.
---      x_event_id      - Event id
---      x_reason        - Text to associate with event.
---
--- Returns:
---     0 if event was already in queue, 1 otherwise.
--- ----------------------------------------------------------------------
-begin
-    insert into pgq.failed_queue (ev_failed_reason, ev_failed_time,
-        ev_id, ev_time, ev_txid, ev_owner, ev_retry, ev_type, ev_data,
-        ev_extra1, ev_extra2, ev_extra3, ev_extra4)
-    select x_reason, now(),
-           ev_id, ev_time, NULL, sub_id, coalesce(ev_retry, 0),
-           ev_type, ev_data, ev_extra1, ev_extra2, ev_extra3, ev_extra4
-      from pgq.get_batch_events(x_batch_id),
-           pgq.subscription
-     where sub_batch = x_batch_id
-       and ev_id = x_event_id;
-    if not found then
-        raise exception 'event not found';
-    end if;
-    return 1;
-
--- dont worry if the event is already in queue
-exception
-    when unique_violation then
-        return 0;
-end;
-$$ language plpgsql security definer;
-
diff --git a/sql/pgq/functions/pgq.failed_queue.sql b/sql/pgq/functions/pgq.failed_queue.sql
deleted file mode 100644
index 0ae02043..00000000
--- a/sql/pgq/functions/pgq.failed_queue.sql
+++ /dev/null
@@ -1,201 +0,0 @@
-
-create or replace function pgq.failed_event_list(
-    x_queue_name text,
-    x_consumer_name text)
-returns setof pgq.failed_queue as $$ 
--- ----------------------------------------------------------------------
--- Function: pgq.failed_event_list(2)
---
---      Get list of all failed events for one consumer.
---
--- Parameters:
---      x_queue_name        - Queue name
---      x_consumer_name     - Consumer name
---
--- Returns:
---      List of failed events.
--- ----------------------------------------------------------------------
-declare 
-    rec pgq.failed_queue%rowtype; 
-begin 
-    for rec in
-        select fq.*
-          from pgq.failed_queue fq, pgq.consumer,
-               pgq.queue, pgq.subscription
-         where queue_name = x_queue_name
-           and co_name = x_consumer_name
-           and sub_consumer = co_id
-           and sub_queue = queue_id
-           and ev_owner = sub_id
-        order by ev_id
-    loop
-        return next rec; 
-    end loop; 
-    return;
-end; 
-$$ language plpgsql security definer;
-
-create or replace function pgq.failed_event_list(
-    x_queue_name text,
-    x_consumer_name text,
-    x_count integer,
-    x_offset integer)
-returns setof pgq.failed_queue as $$ 
--- ----------------------------------------------------------------------
--- Function: pgq.failed_event_list(4)
---
---      Get list of failed events, from offset and specific count.
---
--- Parameters:
---      x_queue_name        - Queue name
---      x_consumer_name     - Consumer name
---      x_count             - Max amount of events to fetch
---      x_offset            - From this offset
---
--- Returns:
---      List of failed events.
--- ----------------------------------------------------------------------
-declare 
-    rec pgq.failed_queue%rowtype; 
-begin 
-    for rec in
-        select fq.*
-          from pgq.failed_queue fq, pgq.consumer,
-               pgq.queue, pgq.subscription
-         where queue_name = x_queue_name
-           and co_name = x_consumer_name
-           and sub_consumer = co_id
-           and sub_queue = queue_id
-           and ev_owner = sub_id
-        order by ev_id
-        limit x_count
-        offset x_offset
-    loop
-        return next rec; 
-    end loop; 
-    return;
-end; 
-$$ language plpgsql security definer;
-
-create or replace function pgq.failed_event_count(
-    x_queue_name text,
-    x_consumer_name text)
-returns integer as $$ 
--- ----------------------------------------------------------------------
--- Function: pgq.failed_event_count(2)
---
---      Get size of failed event queue.
---
--- Parameters:
---      x_queue_name        - Queue name
---      x_consumer_name     - Consumer name
---
--- Returns:
---      Number of failed events in failed event queue.
--- ----------------------------------------------------------------------
-declare 
-    ret integer;
-begin 
-    select count(1) into ret
-      from pgq.failed_queue, pgq.consumer, pgq.queue, pgq.subscription
-     where queue_name = x_queue_name
-       and co_name = x_consumer_name
-       and sub_queue = queue_id
-       and sub_consumer = co_id
-       and ev_owner = sub_id;
-    return ret;
-end; 
-$$ language plpgsql security definer;
-
-create or replace function pgq.failed_event_delete(
-    x_queue_name text,
-    x_consumer_name text,
-    x_event_id bigint)
-returns integer as $$ 
--- ----------------------------------------------------------------------
--- Function: pgq.failed_event_delete(3)
---
---      Delete specific event from failed event queue.
---
--- Parameters:
---      x_queue_name        - Queue name
---      x_consumer_name     - Consumer name
---      x_event_id          - Event ID
---
--- Returns:
---      nothing
--- ----------------------------------------------------------------------
-declare 
-    x_sub_id integer;
-begin 
-    select sub_id into x_sub_id
-      from pgq.subscription, pgq.consumer, pgq.queue
-     where queue_name = x_queue_name
-       and co_name = x_consumer_name
-       and sub_consumer = co_id
-       and sub_queue = queue_id;
-    if not found then
-        raise exception 'no such queue/consumer';
-    end if;
-
-    delete from pgq.failed_queue
-     where ev_owner = x_sub_id
-       and ev_id = x_event_id;
-    if not found then
-        raise exception 'event not found';
-    end if;
-
-    return 1;
-end; 
-$$ language plpgsql security definer;
-
-create or replace function pgq.failed_event_retry(
-    x_queue_name text,
-    x_consumer_name text,
-    x_event_id bigint)
-returns bigint as $$ 
--- ----------------------------------------------------------------------
--- Function: pgq.failed_event_retry(3)
---
---      Insert specific event from failed queue to main queue.
---
--- Parameters:
---      x_queue_name        - Queue name
---      x_consumer_name     - Consumer name
---      x_event_id          - Event ID
---
--- Returns:
---      nothing
--- ----------------------------------------------------------------------
-declare 
-    ret         bigint;
-    x_sub_id    integer;
-begin 
-    select sub_id into x_sub_id
-      from pgq.subscription, pgq.consumer, pgq.queue
-     where queue_name = x_queue_name
-       and co_name = x_consumer_name
-       and sub_consumer = co_id
-       and sub_queue = queue_id;
-    if not found then
-        raise exception 'no such queue/consumer';
-    end if;
-
-    select pgq.insert_event_raw(x_queue_name, ev_id, ev_time,
-            ev_owner, ev_retry, ev_type, ev_data,
-            ev_extra1, ev_extra2, ev_extra3, ev_extra4)
-      into ret
-      from pgq.failed_queue, pgq.consumer, pgq.queue
-     where ev_owner = x_sub_id
-       and ev_id = x_event_id;
-    if not found then
-        raise exception 'event not found';
-    end if;
-
-    perform pgq.failed_event_delete(x_queue_name, x_consumer_name, x_event_id);
-
-    return ret;
-end; 
-$$ language plpgsql security definer;
-
-
diff --git a/sql/pgq/functions/pgq.unregister_consumer.sql b/sql/pgq/functions/pgq.unregister_consumer.sql
index 9356bcec..17e83728 100644
--- a/sql/pgq/functions/pgq.unregister_consumer.sql
+++ b/sql/pgq/functions/pgq.unregister_consumer.sql
@@ -41,9 +41,6 @@ begin
     delete from pgq.retry_queue
         where ev_owner = x_sub_id;
 
-    delete from pgq.failed_queue
-        where ev_owner = x_sub_id;
-
     delete from pgq.subscription
         where sub_id = x_sub_id;
 
diff --git a/sql/pgq/sql/pgq_core.sql b/sql/pgq/sql/pgq_core.sql
index eab7081d..3f714568 100644
--- a/sql/pgq/sql/pgq_core.sql
+++ b/sql/pgq/sql/pgq_core.sql
@@ -54,23 +54,10 @@ select ev_id,ev_retry,ev_type,ev_data,ev_extra1,ev_extra2,ev_extra3,ev_extra4
 close acurs;
 end;
 
-select * from pgq.failed_event_list('myqueue', 'consumer');
-
-select pgq.event_failed(3, 1, 'failure test');
-select pgq.event_failed(3, 1, 'failure test');
 select pgq.event_retry(3, 2, 0);
 select pgq.event_retry(3, 2, 0);
 select pgq.finish_batch(3);
 
-select ev_failed_reason, ev_id, ev_txid, ev_retry, ev_type, ev_data
-  from pgq.failed_event_list('myqueue', 'consumer');
-
-select ev_failed_reason, ev_id, ev_txid, ev_retry, ev_type, ev_data
-  from pgq.failed_event_list('myqueue', 'consumer', 0, 1);
-
-select * from pgq.failed_event_count('myqueue', 'consumer');
-select * from pgq.failed_event_delete('myqueue', 'consumer', 0);
-
 select pgq.event_retry_raw('myqueue', 'consumer', now(), 666, now(), 0,
         'rawtest', 'data', null, null, null, null);
 
diff --git a/sql/pgq/structure/func_public.sql b/sql/pgq/structure/func_public.sql
index b6a3d56e..de8b132c 100644
--- a/sql/pgq/structure/func_public.sql
+++ b/sql/pgq/structure/func_public.sql
@@ -21,7 +21,6 @@
 \i functions/pgq.next_batch.sql
 \i functions/pgq.get_batch_events.sql
 \i functions/pgq.get_batch_cursor.sql
-\i functions/pgq.event_failed.sql
 \i functions/pgq.event_retry.sql
 \i functions/pgq.finish_batch.sql
 
@@ -32,7 +31,3 @@
 \i functions/pgq.version.sql
 \i functions/pgq.get_batch_info.sql
 
--- Group: Failed queue browsing
-
-\i functions/pgq.failed_queue.sql
-
diff --git a/sql/pgq/structure/grants.sql b/sql/pgq/structure/grants.sql
index 97a22cbf..d5f8ef1f 100644
--- a/sql/pgq/structure/grants.sql
+++ b/sql/pgq/structure/grants.sql
@@ -7,4 +7,3 @@ grant select on table pgq.queue to public;
 grant select on table pgq.subscription to public;
 grant select on table pgq.event_template to public;
 grant select on table pgq.retry_queue to public;
-grant select on table pgq.failed_queue to public;
diff --git a/sql/pgq/structure/tables.sql b/sql/pgq/structure/tables.sql
index 3d2dfdcc..521d1a9d 100644
--- a/sql/pgq/structure/tables.sql
+++ b/sql/pgq/structure/tables.sql
@@ -218,27 +218,3 @@ alter table pgq.retry_queue alter column ev_owner set not null;
 alter table pgq.retry_queue alter column ev_txid drop not null;
 create index rq_retry_idx on pgq.retry_queue (ev_retry_after);
 
--- ----------------------------------------------------------------------
--- Table: pgq.failed_queue
---
---      Events whose processing failed.
---
--- Columns:
---      ev_failed_reason               - consumer's excuse for not processing
---      ev_failed_time                 - when it was tagged failed
---      ev_queue                       - queue id
---      *                              - same as pgq.event_template
--- ----------------------------------------------------------------------
-create table pgq.failed_queue (
-    ev_failed_reason                   text,
-    ev_failed_time                     timestamptz not null,
-
-    -- all event fields
-    like pgq.event_template,
-
-    constraint fq_pkey primary key (ev_owner, ev_id)
-);
-alter table pgq.failed_queue alter column ev_owner set not null;
-alter table pgq.failed_queue alter column ev_txid drop not null;
-
-