summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMarko Kreen2009-02-13 10:29:12 +0000
committerMarko Kreen2009-02-13 12:21:08 +0000
commit67e3ac04a87179e4283f71a5b8a55e3ddde30791 (patch)
tree8c8bb31d9ded22b010ffc89142eebabc15bf2a90
parent5521e5fc2f399a923fa7fb313bbe797cfa0d5baa (diff)
sql/pgq update
structure: - event_seq field for pgq.tick table, to store last value from event_id_seq - 'disabled' field for pgq.queue, which can disable any direct inserts to queue. inserts are allowed only if session_role = 'replica' functions: - pgq.next_batch_info() as next_batch() but returns full details - pgq.ticker(): now contains ticker logic that previously was located in python code - it uses the event_seq field in tick table to know about last sequence value - pgq.insert_event_raw() - check disabled flag - pgq.seq_getval() / pgq.seq_setval() functions for safe sequence variable manipulation - remove denytriga, now regular triggers can play the role - use OUT parameters instead of ret types - pgq.force_tick(): silently ignore the request on disabled queues - pgq.maint_retry_events(): fwd port bugfix from 2.1-stable - pgq.maint_rotate_tables(): cleanup from 2.1-stable - pgq.maint_tables_to_vacuum(): add more tables - triggers/common.c: bugfix from 2.1-stable
-rw-r--r--sql/pgq/Makefile6
-rw-r--r--sql/pgq/expected/denytriga.out37
-rw-r--r--sql/pgq/expected/pgq_core.out63
-rw-r--r--sql/pgq/functions/pgq.current_event_table.sql10
-rw-r--r--sql/pgq/functions/pgq.force_tick.sql9
-rw-r--r--sql/pgq/functions/pgq.get_batch_events.sql35
-rw-r--r--sql/pgq/functions/pgq.get_consumer_info.sql120
-rw-r--r--sql/pgq/functions/pgq.get_queue_info.sql81
-rw-r--r--sql/pgq/functions/pgq.maint_retry_events.sql14
-rw-r--r--sql/pgq/functions/pgq.maint_rotate_tables.sql64
-rw-r--r--sql/pgq/functions/pgq.maint_tables_to_vacuum.sql57
-rw-r--r--sql/pgq/functions/pgq.next_batch.sql114
-rw-r--r--sql/pgq/functions/pgq.seq_funcs.sql38
-rw-r--r--sql/pgq/functions/pgq.ticker.sql106
-rw-r--r--sql/pgq/lowlevel/insert_event.c13
-rw-r--r--sql/pgq/sql/denytriga.sql18
-rw-r--r--sql/pgq/sql/pgq_core.sql9
-rw-r--r--sql/pgq/sql/pgq_init.sql3
-rw-r--r--sql/pgq/structure/install.sql1
-rw-r--r--sql/pgq/structure/tables.sql37
-rw-r--r--sql/pgq/structure/types.sql39
-rw-r--r--sql/pgq/structure/uninstall_pgq.sql2
-rw-r--r--sql/pgq/triggers/Makefile2
-rw-r--r--sql/pgq/triggers/common.c51
-rw-r--r--sql/pgq/triggers/common.h2
-rw-r--r--sql/pgq/triggers/denytriga.c75
-rw-r--r--sql/pgq/triggers/logtriga.c4
-rw-r--r--sql/pgq/triggers/logutriga.c3
-rw-r--r--sql/pgq/triggers/pgq_triggers.sql.in31
-rw-r--r--sql/pgq/triggers/sqltriga.c7
30 files changed, 586 insertions, 465 deletions
diff --git a/sql/pgq/Makefile b/sql/pgq/Makefile
index 26c15426..37facee4 100644
--- a/sql/pgq/Makefile
+++ b/sql/pgq/Makefile
@@ -7,9 +7,9 @@ DATA = structure/uninstall_pgq.sql
LDRS = structure/func_internal.sql structure/func_public.sql structure/triggers.sql
FUNCS = $(shell sed -n -e '/^\\/{s/\\i //;p}' $(LDRS))
SRCS = structure/tables.sql structure/grants.sql structure/install.sql \
- structure/types.sql structure/uninstall_pgq.sql $(FUNCS)
+ structure/uninstall_pgq.sql $(FUNCS)
-REGRESS = pgq_init pgq_core logutriga sqltriga denytriga
+REGRESS = pgq_init pgq_core logutriga sqltriga
REGRESS_OPTS = --load-language=plpgsql
include ../../config.mak
@@ -51,7 +51,7 @@ pgq.upgrade.sql: $(SRCS)
dox: cleandox $(SRCS)
mkdir -p docs/html
mkdir -p docs/sql
- $(CATSQL) --ndoc structure/tables.sql structure/types.sql > docs/sql/schema.sql
+ $(CATSQL) --ndoc structure/tables.sql > docs/sql/schema.sql
$(CATSQL) --ndoc structure/func_public.sql > docs/sql/external.sql
$(CATSQL) --ndoc structure/func_internal.sql > docs/sql/internal.sql
$(CATSQL) --ndoc structure/triggers.sql > docs/sql/triggers.sql
diff --git a/sql/pgq/expected/denytriga.out b/sql/pgq/expected/denytriga.out
deleted file mode 100644
index 4dac73d4..00000000
--- a/sql/pgq/expected/denytriga.out
+++ /dev/null
@@ -1,37 +0,0 @@
-create table denytest (
- id integer
-);
-create trigger denytrg after insert or update or delete
-on denytest for each row execute procedure pgq.denytriga('baz');
-insert into denytest values (1); -- must fail
-ERROR: action denied
-select pgq.set_connection_context('foo');
- set_connection_context
-------------------------
-
-(1 row)
-
-insert into denytest values (1); -- must fail
-ERROR: action denied
-select pgq.set_connection_context('baz');
- set_connection_context
-------------------------
-
-(1 row)
-
-insert into denytest values (1); -- must succeed
-select pgq.set_connection_context(null);
- set_connection_context
-------------------------
-
-(1 row)
-
-delete from denytest; -- must fail
-ERROR: action denied
-select pgq.set_connection_context('baz');
- set_connection_context
-------------------------
-
-(1 row)
-
-delete from denytest; -- must succeed
diff --git a/sql/pgq/expected/pgq_core.out b/sql/pgq/expected/pgq_core.out
index 21baed32..e90786ca 100644
--- a/sql/pgq/expected/pgq_core.out
+++ b/sql/pgq/expected/pgq_core.out
@@ -50,6 +50,7 @@ select pgq.register_consumer('myqueue', 'consumer');
1
(1 row)
+update pgq.queue set queue_ticker_max_lag = '0', queue_ticker_idle_period = '0';
select pgq.next_batch('myqueue', 'consumer');
next_batch
------------
@@ -80,10 +81,10 @@ select pgq.next_batch('myqueue', 'consumer');
1
(1 row)
-select queue_name, consumer_name, prev_tick_id, tick_id, lag from pgq.get_batch_info(1);
- queue_name | consumer_name | prev_tick_id | tick_id | lag
-------------+---------------+--------------+---------+-------------
- myqueue | consumer | 1 | 2 | @ 0.00 secs
+select queue_name, consumer_name, prev_tick_id, tick_id, lag < '30 seconds' as lag_exists from pgq.get_batch_info(1);
+ queue_name | consumer_name | prev_tick_id | tick_id | lag_exists
+------------+---------------+--------------+---------+------------
+ myqueue | consumer | 1 | 2 | t
(1 row)
select queue_name, queue_ntables, queue_cur_table, queue_rotation_period,
@@ -93,7 +94,7 @@ select queue_name, queue_ntables, queue_cur_table, queue_rotation_period,
from pgq.get_queue_info() order by 1;
queue_name | queue_ntables | queue_cur_table | queue_rotation_period | switch_time_exists | queue_external_ticker | queue_ticker_max_count | queue_ticker_max_lag | queue_ticker_idle_period | ticker_lag_exists
------------+---------------+-----------------+-----------------------+--------------------+-----------------------+------------------------+----------------------+--------------------------+-------------------
- myqueue | 3 | 0 | @ 2 hours | t | f | 500 | @ 3 secs | @ 1 min | t
+ myqueue | 3 | 0 | @ 2 hours | t | f | 500 | @ 0 | @ 0 | t
(1 row)
select queue_name, consumer_name, lag < '30 seconds' as lag_exists,
@@ -290,6 +291,56 @@ select pgq.force_tick('myqueue');
select nextval(queue_event_seq) from pgq.queue where queue_name = 'myqueue';
nextval
---------
- 1006
+ 2006
+(1 row)
+
+create sequence tmptest_seq;
+select pgq.seq_getval('tmptest_seq');
+ seq_getval
+------------
+ 1
+(1 row)
+
+select pgq.seq_setval('tmptest_seq', 10);
+ seq_setval
+------------
+ 10
+(1 row)
+
+select pgq.seq_setval('tmptest_seq', 5);
+ seq_setval
+------------
+ 10
+(1 row)
+
+select pgq.seq_setval('tmptest_seq', 15);
+ seq_setval
+------------
+ 15
+(1 row)
+
+select pgq.seq_getval('tmptest_seq');
+ seq_getval
+------------
+ 15
+(1 row)
+
+-- test disabled
+select pgq.insert_event('myqueue', 'test', 'event');
+ insert_event
+--------------
+ 2007
+(1 row)
+
+update pgq.queue set queue_disable_insert = true where queue_name = 'myqueue';
+select pgq.insert_event('myqueue', 'test', 'event');
+ERROR: Insert into queue disallowed
+CONTEXT: PL/pgSQL function "insert_event" line 19 at RETURN
+PL/pgSQL function "insert_event" line 15 at RETURN
+update pgq.queue set queue_disable_insert = false where queue_name = 'myqueue';
+select pgq.insert_event('myqueue', 'test', 'event');
+ insert_event
+--------------
+ 2009
(1 row)
diff --git a/sql/pgq/functions/pgq.current_event_table.sql b/sql/pgq/functions/pgq.current_event_table.sql
index 5349bb7e..3e4d74fb 100644
--- a/sql/pgq/functions/pgq.current_event_table.sql
+++ b/sql/pgq/functions/pgq.current_event_table.sql
@@ -18,12 +18,20 @@ returns text as $$
-- ----------------------------------------------------------------------
declare
res text;
+ disabled boolean;
begin
- select queue_data_pfx || '_' || queue_cur_table into res
+ select queue_data_pfx || '_' || queue_cur_table,
+ queue_disable_insert
+ into res, disabled
from pgq.queue where queue_name = x_queue_name;
if not found then
raise exception 'Event queue not found';
end if;
+ if disabled then
+ if current_setting('session_replication_role') <> 'replica' then
+ raise exception 'Writing to queue disabled';
+ end if;
+ end if;
return res;
end;
$$ language plpgsql; -- no perms needed
diff --git a/sql/pgq/functions/pgq.force_tick.sql b/sql/pgq/functions/pgq.force_tick.sql
index 0ea098d9..c1550d9a 100644
--- a/sql/pgq/functions/pgq.force_tick.sql
+++ b/sql/pgq/functions/pgq.force_tick.sql
@@ -27,13 +27,14 @@ begin
-- bump seq and get queue id
select queue_id,
setval(queue_event_seq, nextval(queue_event_seq)
- + queue_ticker_max_count * 2) as tmp
+ + queue_ticker_max_count * 2 + 1000) as tmp
into q from pgq.queue
where queue_name = i_queue_name
and not queue_external_ticker;
- if not found then
- raise exception 'queue not found or ticks not allowed';
- end if;
+
+ --if not found then
+ -- raise notice 'queue not found or ticks not allowed';
+ --end if;
-- return last tick id
select tick_id into t from pgq.tick
diff --git a/sql/pgq/functions/pgq.get_batch_events.sql b/sql/pgq/functions/pgq.get_batch_events.sql
index 8166d519..3fadaaf7 100644
--- a/sql/pgq/functions/pgq.get_batch_events.sql
+++ b/sql/pgq/functions/pgq.get_batch_events.sql
@@ -1,5 +1,16 @@
-create or replace function pgq.get_batch_events(x_batch_id bigint)
-returns setof pgq.ret_batch_event as $$
+create or replace function pgq.get_batch_events(
+ in x_batch_id bigint,
+ out ev_id bigint,
+ out ev_time timestamptz,
+ out ev_txid bigint,
+ out ev_retry int4,
+ out ev_type text,
+ out ev_data text,
+ out ev_extra1 text,
+ out ev_extra2 text,
+ out ev_extra3 text,
+ out ev_extra4 text)
+returns setof record as $$
-- ----------------------------------------------------------------------
-- Function: pgq.get_batch_events(1)
--
@@ -11,16 +22,18 @@ returns setof pgq.ret_batch_event as $$
-- Returns:
-- List of events.
-- ----------------------------------------------------------------------
-declare
- rec pgq.ret_batch_event%rowtype;
- sql text;
-begin
- sql := pgq.batch_event_sql(x_batch_id);
- for rec in execute sql loop
- return next rec;
- end loop;
+declare
+ sql text;
+begin
+ sql := pgq.batch_event_sql(x_batch_id);
+ for ev_id, ev_time, ev_txid, ev_retry, ev_type, ev_data,
+ ev_extra1, ev_extra2, ev_extra3, ev_extra4
+ in execute sql
+ loop
+ return next;
+ end loop;
return;
-end;
+end;
$$ language plpgsql; -- no perms needed
diff --git a/sql/pgq/functions/pgq.get_consumer_info.sql b/sql/pgq/functions/pgq.get_consumer_info.sql
index 27dd444d..a101c9b3 100644
--- a/sql/pgq/functions/pgq.get_consumer_info.sql
+++ b/sql/pgq/functions/pgq.get_consumer_info.sql
@@ -1,7 +1,13 @@
--------------------------------------------------------------------------
-create or replace function pgq.get_consumer_info()
-returns setof pgq.ret_consumer_info as $$
+create or replace function pgq.get_consumer_info(
+ out queue_name text,
+ out consumer_name text,
+ out lag interval,
+ out last_seen interval,
+ out last_tick bigint,
+ out current_batch bigint,
+ out next_tick bigint)
+returns setof record as $$
-- ----------------------------------------------------------------------
-- Function: pgq.get_consumer_info(0)
--
@@ -10,73 +16,75 @@ returns setof pgq.ret_consumer_info as $$
-- Returns:
-- See pgq.get_consumer_info(2)
-- ----------------------------------------------------------------------
-declare
- ret pgq.ret_consumer_info%rowtype;
- i record;
begin
- for i in select queue_name from pgq.queue order by 1
+ for queue_name, consumer_name, lag, last_seen,
+ last_tick, current_batch, next_tick
+ in
+ select f.queue_name, f.consumer_name, f.lag, f.last_seen,
+ f.last_tick, f.current_batch, f.next_tick
+ from pgq.get_consumer_info(null, null) f
loop
- for ret in
- select * from pgq.get_consumer_info(i.queue_name)
- loop
- return next ret;
- end loop;
+ return next;
end loop;
return;
end;
$$ language plpgsql security definer;
--------------------------------------------------------------------------
-create or replace function pgq.get_consumer_info(x_queue_name text)
-returns setof pgq.ret_consumer_info as $$
+
+create or replace function pgq.get_consumer_info(
+ in i_queue_name text,
+ out queue_name text,
+ out consumer_name text,
+ out lag interval,
+ out last_seen interval,
+ out last_tick bigint,
+ out current_batch bigint,
+ out next_tick bigint)
+returns setof record as $$
-- ----------------------------------------------------------------------
-- Function: pgq.get_consumer_info(1)
--
--- Returns info about consumers on one particular queue.
---
--- Parameters:
--- x_queue_name - Queue name
+-- Returns info about all consumers on single queue.
--
-- Returns:
-- See pgq.get_consumer_info(2)
-- ----------------------------------------------------------------------
-declare
- ret pgq.ret_consumer_info%rowtype;
- tmp record;
begin
- for tmp in
- select queue_name, co_name
- from pgq.queue, pgq.consumer, pgq.subscription
- where queue_id = sub_queue
- and co_id = sub_consumer
- and queue_name = x_queue_name
- order by 1, 2
+ for queue_name, consumer_name, lag, last_seen,
+ last_tick, current_batch, next_tick
+ in
+ select f.queue_name, f.consumer_name, f.lag, f.last_seen,
+ f.last_tick, f.current_batch, f.next_tick
+ from pgq.get_consumer_info(i_queue_name, null) f
loop
- for ret in
- select * from pgq.get_consumer_info(tmp.queue_name, tmp.co_name)
- loop
- return next ret;
- end loop;
+ return next;
end loop;
return;
end;
$$ language plpgsql security definer;
-------------------------------------------------------------------------
+
create or replace function pgq.get_consumer_info(
- x_queue_name text,
- x_consumer_name text)
-returns setof pgq.ret_consumer_info as $$
+ in i_queue_name text,
+ in i_consumer_name text,
+ out queue_name text,
+ out consumer_name text,
+ out lag interval,
+ out last_seen interval,
+ out last_tick bigint,
+ out current_batch bigint,
+ out next_tick bigint)
+returns setof record as $$
-- ----------------------------------------------------------------------
-- Function: pgq.get_consumer_info(2)
--
-- Get info about particular consumer on particular queue.
--
-- Parameters:
--- x_queue_name - name of a queue.
--- x_consumer_name - name of a consumer
+-- i_queue_name - name of a queue. (null = all)
+-- i_consumer_name - name of a consumer (null = all)
--
-- Returns:
-- queue_name - Queue name
@@ -87,26 +95,24 @@ returns setof pgq.ret_consumer_info as $$
-- current_batch - Current batch ID, if one is active or NULL
-- next_tick - If batch is active, then its final tick.
-- ----------------------------------------------------------------------
-declare
- ret pgq.ret_consumer_info%rowtype;
begin
- for ret in
- select queue_name, co_name,
- current_timestamp - tick_time as lag,
- current_timestamp - sub_active as last_seen,
- sub_last_tick as last_tick,
- sub_batch as current_batch,
- sub_next_tick as next_tick
- from pgq.subscription, pgq.tick, pgq.queue, pgq.consumer
- where tick_id = sub_last_tick
- and queue_id = sub_queue
- and tick_queue = sub_queue
- and co_id = sub_consumer
- and queue_name = x_queue_name
- and co_name = x_consumer_name
+ for queue_name, consumer_name, lag, last_seen,
+ last_tick, current_batch, next_tick
+ in
+ select q.queue_name, c.co_name,
+ current_timestamp - t.tick_time,
+ current_timestamp - s.sub_active,
+ s.sub_last_tick, s.sub_batch, s.sub_next_tick
+ from pgq.subscription s, pgq.tick t, pgq.queue q, pgq.consumer c
+ where t.tick_id = s.sub_last_tick
+ and q.queue_id = s.sub_queue
+ and t.tick_queue = s.sub_queue
+ and c.co_id = s.sub_consumer
+ and (i_queue_name is null or q.queue_name = i_queue_name)
+ and (i_consumer_name is null or c.co_name = i_consumer_name)
order by 1,2
loop
- return next ret;
+ return next;
end loop;
return;
end;
diff --git a/sql/pgq/functions/pgq.get_queue_info.sql b/sql/pgq/functions/pgq.get_queue_info.sql
index 9d15d49e..14906770 100644
--- a/sql/pgq/functions/pgq.get_queue_info.sql
+++ b/sql/pgq/functions/pgq.get_queue_info.sql
@@ -1,5 +1,15 @@
-create or replace function pgq.get_queue_info()
-returns setof pgq.ret_queue_info as $$
+create or replace function pgq.get_queue_info(
+ out queue_name text,
+ out queue_ntables integer,
+ out queue_cur_table integer,
+ out queue_rotation_period interval,
+ out queue_switch_time timestamptz,
+ out queue_external_ticker boolean,
+ out queue_ticker_max_count integer,
+ out queue_ticker_max_lag interval,
+ out queue_ticker_idle_period interval,
+ out ticker_lag interval)
+returns setof record as $$
-- ----------------------------------------------------------------------
-- Function: pgq.get_queue_info(0)
--
@@ -8,22 +18,35 @@ returns setof pgq.ret_queue_info as $$
-- Returns:
-- List of pgq.ret_queue_info records.
-- ----------------------------------------------------------------------
-declare
- q record;
- ret pgq.ret_queue_info%rowtype;
begin
- for q in
- select queue_name from pgq.queue order by 1
+ for queue_name, queue_ntables, queue_cur_table, queue_rotation_period,
+ queue_switch_time, queue_external_ticker, queue_ticker_max_count,
+ queue_ticker_max_lag, queue_ticker_idle_period, ticker_lag
+ in select
+ f.queue_name, f.queue_ntables, f.queue_cur_table, f.queue_rotation_period,
+ f.queue_switch_time, f.queue_external_ticker, f.queue_ticker_max_count,
+ f.queue_ticker_max_lag, f.queue_ticker_idle_period, f.ticker_lag
+ from pgq.get_queue_info(null) f
loop
- select * into ret from pgq.get_queue_info(q.queue_name);
- return next ret;
+ return next;
end loop;
return;
end;
-$$ language plpgsql security definer;
+$$ language plpgsql;
-create or replace function pgq.get_queue_info(qname text)
-returns pgq.ret_queue_info as $$
+create or replace function pgq.get_queue_info(
+ in i_queue_name text,
+ out queue_name text,
+ out queue_ntables integer,
+ out queue_cur_table integer,
+ out queue_rotation_period interval,
+ out queue_switch_time timestamptz,
+ out queue_external_ticker boolean,
+ out queue_ticker_max_count integer,
+ out queue_ticker_max_lag interval,
+ out queue_ticker_idle_period interval,
+ out ticker_lag interval)
+returns setof record as $$
-- ----------------------------------------------------------------------
-- Function: pgq.get_queue_info(1)
--
@@ -32,20 +55,26 @@ returns pgq.ret_queue_info as $$
-- Returns:
-- One pgq.ret_queue_info record.
-- ----------------------------------------------------------------------
-declare
- ret pgq.ret_queue_info%rowtype;
begin
- select queue_name, queue_ntables, queue_cur_table,
- queue_rotation_period, queue_switch_time,
- queue_external_ticker,
- queue_ticker_max_count, queue_ticker_max_lag,
- queue_ticker_idle_period,
- (select current_timestamp - tick_time
- from pgq.tick where tick_queue = queue_id
- order by tick_queue desc, tick_id desc limit 1
- ) as ticker_lag
- into ret from pgq.queue where queue_name = qname;
- return ret;
+ for queue_name, queue_ntables, queue_cur_table, queue_rotation_period,
+ queue_switch_time, queue_external_ticker, queue_ticker_max_count,
+ queue_ticker_max_lag, queue_ticker_idle_period, ticker_lag
+ in select
+ q.queue_name, q.queue_ntables, q.queue_cur_table,
+ q.queue_rotation_period, q.queue_switch_time,
+ q.queue_external_ticker,
+ q.queue_ticker_max_count, q.queue_ticker_max_lag,
+ q.queue_ticker_idle_period,
+ (select current_timestamp - tick_time
+ from pgq.tick where tick_queue = queue_id
+ order by tick_queue desc, tick_id desc limit 1)
+ from pgq.queue q
+ where (i_queue_name is null or q.queue_name = i_queue_name)
+ order by q.queue_name
+ loop
+ return next;
+ end loop;
+ return;
end;
-$$ language plpgsql security definer;
+$$ language plpgsql;
diff --git a/sql/pgq/functions/pgq.maint_retry_events.sql b/sql/pgq/functions/pgq.maint_retry_events.sql
index f3038b86..8e4bfa4b 100644
--- a/sql/pgq/functions/pgq.maint_retry_events.sql
+++ b/sql/pgq/functions/pgq.maint_retry_events.sql
@@ -8,9 +8,6 @@ returns integer as $$
-- It moves small amount at a time. It should be called
-- until it returns 0
--
--- Parameters:
--- arg - desc
---
-- Returns:
-- Number of events processed.
-- ----------------------------------------------------------------------
@@ -20,10 +17,9 @@ declare
begin
cnt := 0;
for rec in
- select pgq.insert_event_raw(queue_name,
- ev_id, ev_time, ev_owner, ev_retry, ev_type, ev_data,
- ev_extra1, ev_extra2, ev_extra3, ev_extra4),
- ev_owner, ev_id
+ select queue_name,
+ ev_id, ev_time, ev_owner, ev_retry, ev_type, ev_data,
+ ev_extra1, ev_extra2, ev_extra3, ev_extra4
from pgq.retry_queue, pgq.queue, pgq.subscription
where ev_retry_after <= current_timestamp
and sub_id = ev_owner
@@ -32,6 +28,10 @@ begin
limit 10
loop
cnt := cnt + 1;
+ perform pgq.insert_event_raw(rec.queue_name,
+ rec.ev_id, rec.ev_time, rec.ev_owner, rec.ev_retry,
+ rec.ev_type, rec.ev_data, rec.ev_extra1, rec.ev_extra2,
+ rec.ev_extra3, rec.ev_extra4);
delete from pgq.retry_queue
where ev_owner = rec.ev_owner
and ev_id = rec.ev_id;
diff --git a/sql/pgq/functions/pgq.maint_rotate_tables.sql b/sql/pgq/functions/pgq.maint_rotate_tables.sql
index b52a5025..d24f09c8 100644
--- a/sql/pgq/functions/pgq.maint_rotate_tables.sql
+++ b/sql/pgq/functions/pgq.maint_rotate_tables.sql
@@ -9,13 +9,15 @@ returns integer as $$
-- i_queue_name - Name of the queue
--
-- Returns:
--- nothing
+-- 1 if rotation happened, otherwise 0.
-- ----------------------------------------------------------------------
declare
- badcnt integer;
- cf record;
- nr integer;
- tbl text;
+ badcnt integer;
+ cf record;
+ nr integer;
+ tbl text;
+ lowest_tick_id int8;
+ lowest_xmin int8;
begin
-- check if needed and load record
select * from pgq.queue into cf
@@ -28,22 +30,26 @@ begin
return 0;
end if;
- -- check if any consumer is on previous table
- select coalesce(count(*), 0) into badcnt
- from pgq.subscription, pgq.tick
- where txid_snapshot_xmin(tick_snapshot) < cf.queue_switch_step2
- and sub_queue = cf.queue_id
- and tick_queue = cf.queue_id
- and tick_id = (select tick_id from pgq.tick
- where tick_id < sub_last_tick
- and tick_queue = sub_queue
- order by tick_queue desc, tick_id desc
- limit 1);
- if badcnt > 0 then
- return 0;
+ -- find lowest tick for that queue
+ select min(sub_last_tick) into lowest_tick_id
+ from pgq.subscription
+ where sub_queue = cf.queue_id;
+
+ -- if some consumer exists
+ if lowest_tick_id is not null then
+ -- is the slowest one still on previous table?
+ select txid_snapshot_xmin(tick_snapshot) into lowest_xmin
+ from pgq.tick
+ where tick_queue = cf.queue_id
+ and tick_id = lowest_tick_id;
+ if lowest_xmin <= cf.queue_switch_step2 then
+ return 0; -- skip rotation then
+ end if;
end if;
- -- all is fine, calc next table number
+ -- nobody on previous table, we can rotate
+
+ -- calc next table number and name
nr := cf.queue_cur_table + 1;
if nr = cf.queue_ntables then
nr := 0;
@@ -57,7 +63,7 @@ begin
execute 'truncate ' || tbl;
exception
when lock_not_available then
- raise warning 'truncate of % failed, skipping rotate', tbl;
+ -- cannot truncate, skipping rotate
return 0;
end;
@@ -69,7 +75,14 @@ begin
queue_switch_step2 = NULL
where queue_id = cf.queue_id;
- -- clean ticks - avoid partial batches
+ -- Clean ticks by using step2 txid from previous rotation.
+ -- That should keep all ticks for all batches that are completely
+ -- in old table. This keeps them for longer than needed, but:
+ -- 1. we want the pgq.tick table to be big, to avoid Postgres
+ -- accitentally switching to seqscans on that.
+ -- 2. that way we guarantee to consumers that they an be moved
+ -- back on the queue at least for one rotation_period.
+ -- (may help in disaster recovery)
delete from pgq.tick
where tick_queue = cf.queue_id
and txid_snapshot_xmin(tick_snapshot) < cf.queue_switch_step2;
@@ -78,16 +91,15 @@ begin
end;
$$ language plpgsql; -- need admin access
+
+create or replace function pgq.maint_rotate_tables_step2()
+returns integer as $$
-- ----------------------------------------------------------------------
-- Function: pgq.maint_rotate_tables_step2(0)
--
--- It tag rotation as finished where needed. It should be
+-- Stores the txid when the rotation was visible. It should be
-- called in separate transaction than pgq.maint_rotate_tables_step1()
-- ----------------------------------------------------------------------
-create or replace function pgq.maint_rotate_tables_step2()
-returns integer as $$
--- visibility tracking. this should run in separate
--- tranaction than step1
begin
update pgq.queue
set queue_switch_step2 = txid_current()
diff --git a/sql/pgq/functions/pgq.maint_tables_to_vacuum.sql b/sql/pgq/functions/pgq.maint_tables_to_vacuum.sql
index b2469d0c..deebb356 100644
--- a/sql/pgq/functions/pgq.maint_tables_to_vacuum.sql
+++ b/sql/pgq/functions/pgq.maint_tables_to_vacuum.sql
@@ -11,35 +11,40 @@ returns setof text as $$
-- List of table names.
-- ----------------------------------------------------------------------
declare
- row record;
+ scm text;
+ tbl text;
+ fqname text;
begin
- return next 'pgq.subscription';
- return next 'pgq.consumer';
- return next 'pgq.queue';
- return next 'pgq.tick';
- return next 'pgq.retry_queue';
-
- -- include also txid, pgq_ext and londiste tables if they exist
- for row in
- select n.nspname as scm, t.relname as tbl
- from pg_class t, pg_namespace n
- where n.oid = t.relnamespace
- and n.nspname = 'txid' and t.relname = 'epoch'
- union all
- select n.nspname as scm, t.relname as tbl
- from pg_class t, pg_namespace n
- where n.oid = t.relnamespace
- and n.nspname = 'londiste' and t.relname = 'completed'
- union all
- select n.nspname as scm, t.relname as tbl
- from pg_class t, pg_namespace n
- where n.oid = t.relnamespace
- and n.nspname = 'pgq_ext'
- and t.relname in ('completed_tick', 'completed_batch', 'completed_event', 'partial_batch')
+ for scm, tbl in values
+ ('pgq', 'subscription'),
+ ('pgq', 'consumer'),
+ ('pgq', 'queue'),
+ ('pgq', 'tick'),
+ ('pgq', 'retry_queue'),
+ ('pgq_ext', 'completed_tick'),
+ ('pgq_ext', 'completed_batch'),
+ ('pgq_ext', 'completed_event'),
+ ('pgq_ext', 'partial_batch'),
+ --('pgq_node', 'node_location'),
+ --('pgq_node', 'node_info'),
+ ('pgq_node', 'local_state'),
+ --('pgq_node', 'subscriber_info'),
+ --('londiste', 'table_info'),
+ ('londiste', 'seq_info'),
+ --('londiste', 'applied_execute'),
+ --('londiste', 'pending_fkeys'),
+ ('txid', 'epoch'),
+ ('londiste', 'completed')
loop
- return next row.scm || '.' || row.tbl;
+ select n.nspname || '.' || t.relname into fqname
+ from pg_class t, pg_namespace n
+ where n.oid = t.relnamespace
+ and n.nspname = scm
+ and t.relname = tbl;
+ if found then
+ return next fqname;
+ end if;
end loop;
-
return;
end;
$$ language plpgsql;
diff --git a/sql/pgq/functions/pgq.next_batch.sql b/sql/pgq/functions/pgq.next_batch.sql
index 88a40e4d..5f9981b2 100644
--- a/sql/pgq/functions/pgq.next_batch.sql
+++ b/sql/pgq/functions/pgq.next_batch.sql
@@ -1,67 +1,129 @@
-create or replace function pgq.next_batch(x_queue_name text, x_consumer_name text)
-returns bigint as $$
+create or replace function pgq.next_batch_info(
+ in i_queue_name text,
+ in i_consumer_name text,
+ out batch_id int8,
+ out cur_tick_id int8,
+ out prev_tick_id int8,
+ out cur_tick_time timestamptz,
+ out prev_tick_time timestamptz,
+ out cur_tick_event_seq int8,
+ out prev_tick_event_seq int8)
+as $$
-- ----------------------------------------------------------------------
--- Function: pgq.next_batch(2)
+-- Function: pgq.next_batch_info(2)
--
-- Makes next block of events active.
--
-- If it returns NULL, there is no events available in queue.
--- Consumer should sleep a bith then.
+-- Consumer should sleep then.
+--
+-- The values from event_id sequence may give hint how big the
+-- batch may be. But they are inexact, they do not give exact size.
+-- Client *MUST NOT* use them to detect whether the batch contains any
+-- events at all - the values are unfit for that purpose.
--
-- Parameters:
--- x_queue_name - Name of the queue
--- x_consumer_name - Name of the consumer
+-- i_queue_name - Name of the queue
+-- i_consumer_name - Name of the consumer
--
-- Returns:
--- Batch ID or NULL if there are no more events available.
+-- batch_id - Batch ID or NULL if there are no more events available.
+-- cur_tick_id - End tick id.
+-- cur_tick_time - End tick time.
+-- cur_tick_event_seq - Value from event id sequence at the time tick was issued.
+-- prev_tick_id - Start tick id.
+-- prev_tick_time - Start tick time.
+-- prev_tick_event_seq - value from event id sequence at the time tick was issued.
-- ----------------------------------------------------------------------
declare
- next_tick bigint;
- batch_id bigint;
errmsg text;
- sub record;
+ queue_id integer;
+ sub_id integer;
+ cons_id integer;
begin
- select sub_queue, sub_consumer, sub_id, sub_last_tick, sub_batch into sub
- from pgq.queue q, pgq.consumer c, pgq.subscription s
- where q.queue_name = x_queue_name
- and c.co_name = x_consumer_name
+ select s.sub_queue, s.sub_consumer, s.sub_id, s.sub_batch,
+ t1.tick_id, t1.tick_time, t1.tick_event_seq,
+ t2.tick_id, t2.tick_time, t2.tick_event_seq
+ into queue_id, cons_id, sub_id, batch_id,
+ prev_tick_id, prev_tick_time, prev_tick_event_seq,
+ cur_tick_id, cur_tick_time, cur_tick_event_seq
+ from pgq.consumer c,
+ pgq.queue q,
+ pgq.subscription s
+ left join pgq.tick t1
+ on (t1.tick_queue = s.sub_queue
+ and t1.tick_id = s.sub_last_tick)
+ left join pgq.tick t2
+ on (t2.tick_queue = s.sub_queue
+ and t2.tick_id = s.sub_next_tick)
+ where q.queue_name = i_queue_name
+ and c.co_name = i_consumer_name
and s.sub_queue = q.queue_id
and s.sub_consumer = c.co_id;
if not found then
errmsg := 'Not subscriber to queue: '
- || coalesce(x_queue_name, 'NULL')
+ || coalesce(i_queue_name, 'NULL')
|| '/'
- || coalesce(x_consumer_name, 'NULL');
+ || coalesce(i_consumer_name, 'NULL');
raise exception '%', errmsg;
end if;
-- has already active batch
- if sub.sub_batch is not null then
- return sub.sub_batch;
+ if batch_id is not null then
+ return;
end if;
-- find next tick
- select tick_id into next_tick
+ select tick_id, tick_time, tick_event_seq
+ into cur_tick_id, cur_tick_time, cur_tick_event_seq
from pgq.tick
- where tick_id > sub.sub_last_tick
- and tick_queue = sub.sub_queue
+ where tick_id > prev_tick_id
+ and tick_queue = queue_id
order by tick_queue asc, tick_id asc
limit 1;
if not found then
-- nothing to do
- return null;
+ prev_tick_id := null;
+ prev_tick_time := null;
+ prev_tick_event_seq := null;
+ return;
end if;
-- get next batch
batch_id := nextval('pgq.batch_id_seq');
update pgq.subscription
set sub_batch = batch_id,
- sub_next_tick = next_tick,
+ sub_next_tick = cur_tick_id,
sub_active = now()
- where sub_queue = sub.sub_queue
- and sub_consumer = sub.sub_consumer;
- return batch_id;
+ where sub_queue = queue_id
+ and sub_consumer = cons_id;
+ return;
end;
$$ language plpgsql security definer;
+create or replace function pgq.next_batch(
+ in i_queue_name text,
+ in i_consumer_name text)
+returns int8 as $$
+-- ----------------------------------------------------------------------
+-- Function: pgq.next_batch(2)
+--
+-- Old function that returns just batch_id.
+--
+-- Parameters:
+-- i_queue_name - Name of the queue
+-- i_consumer_name - Name of the consumer
+--
+-- Returns:
+-- Batch ID or NULL if there are no more events available.
+-- ----------------------------------------------------------------------
+declare
+ res int8;
+begin
+ select batch_id into res
+ from pgq.next_batch_info(i_queue_name, i_consumer_name);
+ return res;
+end;
+$$ language plpgsql;
+
diff --git a/sql/pgq/functions/pgq.seq_funcs.sql b/sql/pgq/functions/pgq.seq_funcs.sql
index 985f6389..4a31f3a9 100644
--- a/sql/pgq/functions/pgq.seq_funcs.sql
+++ b/sql/pgq/functions/pgq.seq_funcs.sql
@@ -4,7 +4,7 @@ returns bigint as $$
-- ----------------------------------------------------------------------
-- Function: pgq.seq_getval(1)
--
--- read current last_val from seq, without offecting it.
+-- Read current last_val from seq, without affecting it.
--
-- Parameters:
-- i_seq_name - Name of the sequence
@@ -13,9 +13,23 @@ returns bigint as $$
-- last value.
-- ----------------------------------------------------------------------
declare
- res int8;
+ res int8;
+ fqname text;
+ pos integer;
+ s text;
+ n text;
begin
- execute 'select last_value from ' || i_seq_name into res;
+ pos := position('.' in i_seq_name);
+ if pos > 0 then
+ s := substring(i_seq_name for pos - 1);
+ n := substring(i_seq_name from pos + 1);
+ else
+ s := 'public';
+ n := i_seq_name;
+ end if;
+ fqname := quote_ident(s) || '.' || quote_ident(n);
+
+ execute 'select last_value from ' || fqname into res;
return res;
end;
$$ language plpgsql;
@@ -35,11 +49,25 @@ returns bigint as $$
-- current last value.
-- ----------------------------------------------------------------------
declare
- res int8;
+ res int8;
+ fqname text;
+ pos integer;
+ s text;
+ n text;
begin
+ pos := position('.' in i_seq_name);
+ if pos > 0 then
+ s := substring(i_seq_name for pos - 1);
+ n := substring(i_seq_name from pos + 1);
+ else
+ s := 'public';
+ n := i_seq_name;
+ end if;
+ fqname := quote_ident(s) || '.' || quote_ident(n);
+
res := pgq.seq_getval(i_seq_name);
if res < i_new_value then
- perform setval(i_seq_name, i_new_value);
+ perform setval(fqname, i_new_value);
return i_new_value;
end if;
return res;
diff --git a/sql/pgq/functions/pgq.ticker.sql b/sql/pgq/functions/pgq.ticker.sql
index 598f43ec..ebfc907c 100644
--- a/sql/pgq/functions/pgq.ticker.sql
+++ b/sql/pgq/functions/pgq.ticker.sql
@@ -1,4 +1,4 @@
-create or replace function pgq.ticker(i_queue_name text, i_tick_id bigint, i_orig_timestamp timestamptz)
+create or replace function pgq.ticker(i_queue_name text, i_tick_id bigint, i_orig_timestamp timestamptz, i_event_seq bigint)
returns bigint as $$
-- ----------------------------------------------------------------------
-- Function: pgq.ticker(3)
@@ -13,43 +13,32 @@ returns bigint as $$
-- Tick id.
-- ----------------------------------------------------------------------
begin
- insert into pgq.tick (tick_queue, tick_id, tick_time)
- select queue_id, i_tick_id, i_orig_timestamp
+ insert into pgq.tick (tick_queue, tick_id, tick_time, tick_event_seq)
+ select queue_id, i_tick_id, i_orig_timestamp, i_event_seq
from pgq.queue
where queue_name = i_queue_name
and queue_external_ticker;
if not found then
raise exception 'queue not found or external ticker disabled: %', i_queue_name;
end if;
+
+ -- make sure seqs stay current
+ perform pgq.seq_setval(queue_tick_seq, i_tick_id),
+ pgq.seq_setval(queue_event_seq, i_event_seq)
+ from pgq.queue
+ where queue_name = i_queue_name;
+
return i_tick_id;
end;
$$ language plpgsql security definer; -- unsure about access
-create or replace function pgq.ticker(i_queue_name text, i_tick_id bigint)
-returns bigint as $$
--- ----------------------------------------------------------------------
--- Function: pgq.ticker(2)
---
--- External ticker: insert a tick with a particular tick_id.
---
--- Parameters:
--- i_queue_name - Name of the queue
--- i_tick_id - Id of new tick.
---
--- Returns:
--- Tick id.
--- ----------------------------------------------------------------------
-begin
- return pgq.ticker(i_queue_name, i_tick_id, now());
-end;
-$$ language plpgsql security definer; -- unsure about access
create or replace function pgq.ticker(i_queue_name text)
returns bigint as $$
-- ----------------------------------------------------------------------
-- Function: pgq.ticker(1)
--
--- Insert a tick with a tick_id from sequence.
+-- Check if tick is needed for the queue and insert it.
--
-- For pgqadm usage.
--
@@ -57,15 +46,19 @@ returns bigint as $$
-- i_queue_name - Name of the queue
--
-- Returns:
--- Tick id.
+-- Tick id or NULL if no tick was done.
-- ----------------------------------------------------------------------
declare
res bigint;
- ext boolean;
- seq text;
q record;
+ state record;
+ last2 record;
begin
- select queue_id, queue_tick_seq, queue_external_ticker into q
+ select queue_id, queue_tick_seq, queue_external_ticker,
+ queue_ticker_max_count, queue_ticker_max_lag,
+ queue_ticker_idle_period, queue_event_seq,
+ pgq.seq_getval(queue_event_seq) as event_seq
+ into q
from pgq.queue where queue_name = i_queue_name;
if not found then
raise exception 'no such queue';
@@ -75,11 +68,51 @@ begin
raise exception 'This queue has external tick source.';
end if;
- insert into pgq.tick (tick_queue, tick_id)
- values (q.queue_id, nextval(q.queue_tick_seq));
+ -- load state from last tick
+ select now() - tick_time as lag,
+ q.event_seq - tick_event_seq as new_events,
+ tick_id, tick_time
+ into state
+ from pgq.tick
+ where tick_queue = q.queue_id
+ order by tick_queue desc, tick_id desc
+ limit 1;
- res = currval(q.queue_tick_seq);
- return res;
+ if found then
+ if state.new_events > 0 then
+ -- there are new events, should we wait a bit?
+ if state.new_events < q.queue_ticker_max_count
+ and state.lag < q.queue_ticker_max_lag
+ then
+ return NULL;
+ end if;
+ else
+ -- no new events, should we apply idle period?
+ -- check previous event from the last one.
+ select state.tick_time - tick_time as lag
+ into last2
+ from pgq.tick
+ where tick_queue = q.queue_id
+ and tick_id < state.tick_id
+ order by tick_queue desc, tick_id desc
+ limit 1;
+ if found then
+ -- gradually decrease the tick frequency
+ if (state.lag < q.queue_ticker_max_lag / 2)
+ or
+ (state.lag < last2.lag * 2
+ and state.lag < q.queue_ticker_idle_period)
+ then
+ return NULL;
+ end if;
+ end if;
+ end if;
+ end if;
+
+ insert into pgq.tick (tick_queue, tick_id, tick_event_seq)
+ values (q.queue_id, nextval(q.queue_tick_seq), q.event_seq);
+
+ return currval(q.queue_tick_seq);
end;
$$ language plpgsql security definer; -- unsure about access
@@ -94,9 +127,18 @@ create or replace function pgq.ticker() returns bigint as $$
-- ----------------------------------------------------------------------
declare
res bigint;
+ q record;
begin
- select count(pgq.ticker(queue_name)) into res
- from pgq.queue where not queue_external_ticker;
+ res := 0;
+ for q in
+ select queue_name from pgq.queue
+ where not queue_external_ticker
+ order by queue_name
+ loop
+ if pgq.ticker(q.queue_name) > 0 then
+ res := res + 1;
+ end if;
+ end loop;
return res;
end;
$$ language plpgsql security definer;
diff --git a/sql/pgq/lowlevel/insert_event.c b/sql/pgq/lowlevel/insert_event.c
index 0efbba74..5431c0cc 100644
--- a/sql/pgq/lowlevel/insert_event.c
+++ b/sql/pgq/lowlevel/insert_event.c
@@ -46,12 +46,14 @@ PG_FUNCTION_INFO_V1(pgq_insert_event_raw);
*/
#define QUEUE_SQL \
"select queue_id::int4, queue_data_pfx::text," \
- " queue_cur_table::int4, nextval(queue_event_seq)::int8 " \
+ " queue_cur_table::int4, nextval(queue_event_seq)::int8," \
+ " queue_disable_insert::bool" \
" from pgq.queue where queue_name = $1"
#define COL_QUEUE_ID 1
#define COL_PREFIX 2
#define COL_TBLNO 3
#define COL_EVENT_ID 4
+#define COL_DISABLED 5
/*
* Plan cache entry in HTAB.
@@ -70,6 +72,7 @@ struct QueueState {
int cur_table;
char *table_prefix;
Datum next_event_id;
+ bool disabled;
};
/*
@@ -193,6 +196,9 @@ static void load_queue_info(Datum queue_name, struct QueueState *state)
state->next_event_id = SPI_getbinval(row, desc, COL_EVENT_ID, &isnull);
if (isnull)
elog(ERROR, "Seq name NULL");
+ state->disabled = SPI_getbinval(row, desc, COL_DISABLED, &isnull);
+ if (isnull)
+ elog(ERROR, "insert_disabled NULL");
}
/*
@@ -221,7 +227,7 @@ pgq_insert_event_raw(PG_FUNCTION_ARGS)
int i, res;
if (PG_NARGS() < 6)
- elog(ERROR, "too few args");
+ elog(ERROR, "Need at least 6 arguments");
if (PG_ARGISNULL(0))
elog(ERROR, "Queue name must not be NULL");
@@ -232,6 +238,9 @@ pgq_insert_event_raw(PG_FUNCTION_ARGS)
load_queue_info(PG_GETARG_DATUM(0), &state);
+ if (state.disabled)
+ elog(ERROR, "Insert into queue disallowed");
+
if (PG_ARGISNULL(1))
ev_id = state.next_event_id;
else
diff --git a/sql/pgq/sql/denytriga.sql b/sql/pgq/sql/denytriga.sql
deleted file mode 100644
index b2cc3261..00000000
--- a/sql/pgq/sql/denytriga.sql
+++ /dev/null
@@ -1,18 +0,0 @@
-
-create table denytest (
- id integer
-);
-
-create trigger denytrg after insert or update or delete
-on denytest for each row execute procedure pgq.denytriga('baz');
-
-insert into denytest values (1); -- must fail
-select pgq.set_connection_context('foo');
-insert into denytest values (1); -- must fail
-select pgq.set_connection_context('baz');
-insert into denytest values (1); -- must succeed
-select pgq.set_connection_context(null);
-delete from denytest; -- must fail
-select pgq.set_connection_context('baz');
-delete from denytest; -- must succeed
-
diff --git a/sql/pgq/sql/pgq_core.sql b/sql/pgq/sql/pgq_core.sql
index 247b4a64..1d655fb6 100644
--- a/sql/pgq/sql/pgq_core.sql
+++ b/sql/pgq/sql/pgq_core.sql
@@ -9,13 +9,14 @@ select pgq.drop_queue('tmpqueue');
select pgq.create_queue('myqueue');
select pgq.register_consumer('myqueue', 'consumer');
+update pgq.queue set queue_ticker_max_lag = '0', queue_ticker_idle_period = '0';
select pgq.next_batch('myqueue', 'consumer');
select pgq.next_batch('myqueue', 'consumer');
select pgq.ticker();
select pgq.next_batch('myqueue', 'consumer');
select pgq.next_batch('myqueue', 'consumer');
-select queue_name, consumer_name, prev_tick_id, tick_id, lag from pgq.get_batch_info(1);
+select queue_name, consumer_name, prev_tick_id, tick_id, lag < '30 seconds' as lag_exists from pgq.get_batch_info(1);
select queue_name, queue_ntables, queue_cur_table, queue_rotation_period,
queue_switch_time <= now() as switch_time_exists,
@@ -85,4 +86,10 @@ select pgq.seq_setval('tmptest_seq', 5);
select pgq.seq_setval('tmptest_seq', 15);
select pgq.seq_getval('tmptest_seq');
+-- test disabled
+select pgq.insert_event('myqueue', 'test', 'event');
+update pgq.queue set queue_disable_insert = true where queue_name = 'myqueue';
+select pgq.insert_event('myqueue', 'test', 'event');
+update pgq.queue set queue_disable_insert = false where queue_name = 'myqueue';
+select pgq.insert_event('myqueue', 'test', 'event');
diff --git a/sql/pgq/sql/pgq_init.sql b/sql/pgq/sql/pgq_init.sql
index 38dd7918..d14cf6ef 100644
--- a/sql/pgq/sql/pgq_init.sql
+++ b/sql/pgq/sql/pgq_init.sql
@@ -1,7 +1,8 @@
\set ECHO none
\i ../txid/txid.sql
-\i pgq.sql
+-- \i pgq.sql
+\i structure/install.sql
\set ECHO all
diff --git a/sql/pgq/structure/install.sql b/sql/pgq/structure/install.sql
index 0dbc85ed..511aaba7 100644
--- a/sql/pgq/structure/install.sql
+++ b/sql/pgq/structure/install.sql
@@ -1,7 +1,6 @@
\i structure/tables.sql
\i structure/grants.sql
-\i structure/types.sql
\i structure/func_internal.sql
\i structure/func_public.sql
\i structure/triggers.sql
diff --git a/sql/pgq/structure/tables.sql b/sql/pgq/structure/tables.sql
index 57e749c1..43d27287 100644
--- a/sql/pgq/structure/tables.sql
+++ b/sql/pgq/structure/tables.sql
@@ -36,11 +36,11 @@ create schema pgq;
-- co_name - consumer's id for external usage
-- ----------------------------------------------------------------------
create table pgq.consumer (
- co_id serial,
- co_name text not null default 'fooz',
+ co_id serial,
+ co_name text not null default 'fooz',
- constraint consumer_pkey primary key (co_id),
- constraint consumer_name_uq UNIQUE (co_name)
+ constraint consumer_pkey primary key (co_id),
+ constraint consumer_name_uq UNIQUE (co_name)
);
@@ -59,6 +59,7 @@ create table pgq.consumer (
-- queue_switch_step2 - tx after rotation was committed
-- queue_switch_time - time when switch happened
-- queue_external_ticker - ticks come from some external sources
+-- queue_disable_insert - disallow pgq.insert_event()
-- queue_ticker_max_count - batch should not contain more events
-- queue_ticker_max_lag - events should not age more
-- queue_ticker_idle_period - how often to tick when no events happen
@@ -67,8 +68,8 @@ create table pgq.consumer (
-- queue_tick_seq - sequence for tick id's
-- ----------------------------------------------------------------------
create table pgq.queue (
- queue_id serial,
- queue_name text not null,
+ queue_id serial,
+ queue_name text not null,
queue_ntables integer not null default 3,
queue_cur_table integer not null default 0,
@@ -78,6 +79,8 @@ create table pgq.queue (
queue_switch_time timestamptz not null default now(),
queue_external_ticker boolean not null default false,
+ queue_disable_insert boolean not null default false,
+
queue_ticker_max_count integer not null default 500,
queue_ticker_max_lag interval not null default '3 seconds',
queue_ticker_idle_period interval not null default '1 minute',
@@ -86,8 +89,8 @@ create table pgq.queue (
queue_event_seq text not null,
queue_tick_seq text not null,
- constraint queue_pkey primary key (queue_id),
- constraint queue_name_uq unique (queue_name)
+ constraint queue_pkey primary key (queue_id),
+ constraint queue_name_uq unique (queue_name)
);
-- ----------------------------------------------------------------------
@@ -100,14 +103,16 @@ create table pgq.queue (
-- tick_id - ticks id (per-queue)
-- tick_time - time when tick happened
-- tick_snapshot - transaction state
+-- tick_event_seq - last value for event seq
-- ----------------------------------------------------------------------
create table pgq.tick (
tick_queue int4 not null,
tick_id bigint not null,
tick_time timestamptz not null default now(),
tick_snapshot txid_snapshot not null default txid_current_snapshot(),
+ tick_event_seq bigint not null, -- may be NULL on upgraded dbs
- constraint tick_pkey primary key (tick_queue, tick_id),
+ constraint tick_pkey primary key (tick_queue, tick_id),
constraint tick_queue_fkey foreign key (tick_queue)
references pgq.queue (queue_id)
);
@@ -134,16 +139,16 @@ create sequence pgq.batch_id_seq;
-- sub_next_tick - batch end pos
-- ----------------------------------------------------------------------
create table pgq.subscription (
- sub_id serial not null,
- sub_queue int4 not null,
- sub_consumer int4 not null,
- sub_last_tick bigint not null,
+ sub_id serial not null,
+ sub_queue int4 not null,
+ sub_consumer int4 not null,
+ sub_last_tick bigint not null,
sub_active timestamptz not null default now(),
sub_batch bigint,
sub_next_tick bigint,
- constraint subscription_pkey primary key (sub_id),
- constraint subscription_ukey unique (sub_queue, sub_consumer),
+ constraint subscription_pkey primary key (sub_id),
+ constraint subscription_ukey unique (sub_queue, sub_consumer),
constraint sub_queue_fkey foreign key (sub_queue)
references pgq.queue (queue_id),
constraint sub_consumer_fkey foreign key (sub_consumer)
@@ -170,7 +175,7 @@ create table pgq.subscription (
-- ev_extra4 - extra data field
-- ----------------------------------------------------------------------
create table pgq.event_template (
- ev_id bigint not null,
+ ev_id bigint not null,
ev_time timestamptz not null,
ev_txid bigint not null default txid_current(),
diff --git a/sql/pgq/structure/types.sql b/sql/pgq/structure/types.sql
deleted file mode 100644
index 9b5c7438..00000000
--- a/sql/pgq/structure/types.sql
+++ /dev/null
@@ -1,39 +0,0 @@
-
-create type pgq.ret_queue_info as (
- queue_name text,
- queue_ntables integer,
- queue_cur_table integer,
- queue_rotation_period interval,
- queue_switch_time timestamptz,
- queue_external_ticker boolean,
- queue_ticker_max_count integer,
- queue_ticker_max_lag interval,
- queue_ticker_idle_period interval,
- ticker_lag interval
-);
-
-create type pgq.ret_consumer_info as (
- queue_name text,
- consumer_name text,
- lag interval,
- last_seen interval,
- last_tick bigint,
- current_batch bigint,
- next_tick bigint
-);
-
-create type pgq.ret_batch_event as (
- ev_id bigint,
- ev_time timestamptz,
-
- ev_txid bigint,
- ev_retry int4,
-
- ev_type text,
- ev_data text,
- ev_extra1 text,
- ev_extra2 text,
- ev_extra3 text,
- ev_extra4 text
-);
-
diff --git a/sql/pgq/structure/uninstall_pgq.sql b/sql/pgq/structure/uninstall_pgq.sql
index 4ab28b30..61545b75 100644
--- a/sql/pgq/structure/uninstall_pgq.sql
+++ b/sql/pgq/structure/uninstall_pgq.sql
@@ -1,4 +1,4 @@
-- brute-force uninstall
-drop schema cascade pgq;
+drop schema pgq cascade;
diff --git a/sql/pgq/triggers/Makefile b/sql/pgq/triggers/Makefile
index 32ac6a05..9d5746a4 100644
--- a/sql/pgq/triggers/Makefile
+++ b/sql/pgq/triggers/Makefile
@@ -2,7 +2,7 @@
include ../../../config.mak
MODULE_big = pgq_triggers
-SRCS = logtriga.c logutriga.c sqltriga.c denytriga.c \
+SRCS = logtriga.c logutriga.c sqltriga.c \
common.c makesql.c stringutil.c
OBJS = $(SRCS:.c=.o)
DATA_built = pgq_triggers.sql
diff --git a/sql/pgq/triggers/common.c b/sql/pgq/triggers/common.c
index cd27ea19..7783d73c 100644
--- a/sql/pgq/triggers/common.c
+++ b/sql/pgq/triggers/common.c
@@ -198,11 +198,12 @@ full_reset(void)
}
/*
- * Create new plan for insertion into current queue table.
+ * Fill table information in hash table.
*/
-static void
-fill_tbl_info(Relation rel, struct PgqTableInfo *info)
+static struct PgqTableInfo *
+fill_tbl_info(Relation rel)
{
+ struct PgqTableInfo *info;
StringInfo pkeys;
Datum values[1];
const char *name = pgq_find_table_name(rel);
@@ -211,12 +212,24 @@ fill_tbl_info(Relation rel, struct PgqTableInfo *info)
bool isnull;
int res, i, attno;
+ /* load pkeys */
values[0] = ObjectIdGetDatum(rel->rd_id);
res = SPI_execute_plan(pkey_plan, values, NULL, false, 0);
if (res != SPI_OK_SELECT)
elog(ERROR, "pkey_plan exec failed");
- desc = SPI_tuptable->tupdesc;
+ /*
+ * SPI_execute_plan may launch reset callback, thus we need
+ * to load the final position after calling it.
+ */
+ init_module();
+ info = hash_search(tbl_cache_map, &rel->rd_id, HASH_ENTER, NULL);
+
+ /*
+ * Fill info
+ */
+
+ desc = SPI_tuptable->tupdesc;
pkeys = makeStringInfo();
info->n_pkeys = SPI_processed;
info->table_name = MemoryContextStrdup(tbl_cache_ctx, name);
@@ -233,6 +246,8 @@ fill_tbl_info(Relation rel, struct PgqTableInfo *info)
appendStringInfoString(pkeys, name);
}
info->pkey_list = MemoryContextStrdup(tbl_cache_ctx, pkeys->data);
+
+ return info;
}
static void
@@ -263,15 +278,15 @@ static void relcache_reset_cb(Datum arg, Oid relid)
struct PgqTableInfo *
pgq_find_table_info(Relation rel)
{
- struct PgqTableInfo *entry;
- bool did_exist = false;
+ struct PgqTableInfo *entry;
- init_module();
+ init_module();
- entry = hash_search(tbl_cache_map, &rel->rd_id, HASH_ENTER, &did_exist);
- if (!did_exist)
- fill_tbl_info(rel, entry);
- return entry;
+ entry = hash_search(tbl_cache_map, &rel->rd_id, HASH_FIND, NULL);
+ if (!entry)
+ entry = fill_tbl_info(rel);
+
+ return entry;
}
static void
@@ -439,3 +454,17 @@ bool pgqtriga_is_pkey(PgqTriggerEvent *ev, TriggerData *tg, int i, int attkind_i
return false;
}
+
+/*
+ * Check if trigger action should be skipped.
+ */
+
+bool pgq_is_logging_disabled(void)
+{
+#if defined(PG_VERSION_NUM) && PG_VERSION_NUM >= 80300
+ if (SessionReplicationRole != SESSION_REPLICATION_ROLE_ORIGIN)
+ return true;
+#endif
+ return false;
+}
+
diff --git a/sql/pgq/triggers/common.h b/sql/pgq/triggers/common.h
index 55319692..5be441b1 100644
--- a/sql/pgq/triggers/common.h
+++ b/sql/pgq/triggers/common.h
@@ -47,6 +47,8 @@ bool pgqtriga_skip_col(PgqTriggerEvent *ev, TriggerData *tg, int i, int attkind_
bool pgqtriga_is_pkey(PgqTriggerEvent *ev, TriggerData *tg, int i, int attkind_idx);
void pgq_insert_tg_event(PgqTriggerEvent *ev);
+bool pgq_is_logging_disabled(void);
+
/* makesql.c */
int pgqtriga_make_sql(PgqTriggerEvent *ev, TriggerData *tg, StringInfo sql);
diff --git a/sql/pgq/triggers/denytriga.c b/sql/pgq/triggers/denytriga.c
deleted file mode 100644
index 45927ef1..00000000
--- a/sql/pgq/triggers/denytriga.c
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * denytriga.c - Dumb deny trigger.
- *
- * Copyright (c) 2008 Marko Kreen, Skype Technologies OÜ
- *
- * Permission to use, copy, modify, and distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
- * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
- * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
- * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
- * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
- * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
- */
-
-#include <postgres.h>
-
-#include <executor/spi.h>
-#include <commands/trigger.h>
-#include <utils/memutils.h>
-
-PG_FUNCTION_INFO_V1(pgq_denytriga);
-Datum pgq_denytriga(PG_FUNCTION_ARGS);
-PG_FUNCTION_INFO_V1(pgq_set_connection_context);
-Datum pgq_set_connection_context(PG_FUNCTION_ARGS);
-
-static char *current_context = NULL;
-
-/*
- * Connection context set.
- */
-
-Datum pgq_set_connection_context(PG_FUNCTION_ARGS)
-{
- char *ctx;
- if (current_context)
- pfree(current_context);
- current_context = NULL;
-
- if (PG_NARGS() > 0 && !PG_ARGISNULL(0)) {
- ctx = DatumGetCString(DirectFunctionCall1(textout, PG_GETARG_DATUM(0)));
- current_context = MemoryContextStrdup(TopMemoryContext, ctx);
- pfree(ctx);
- }
-
- PG_RETURN_VOID();
-}
-
-Datum
-pgq_denytriga(PG_FUNCTION_ARGS)
-{
- TriggerData *tg = (TriggerData *) (fcinfo->context);
-
- if (!CALLED_AS_TRIGGER(fcinfo))
- elog(ERROR, "pgq.denytriga not called as trigger");
- if (!TRIGGER_FIRED_AFTER(tg->tg_event))
- elog(ERROR, "pgq.denytriga must be fired AFTER");
- if (!TRIGGER_FIRED_FOR_ROW(tg->tg_event))
- elog(ERROR, "pgq.denytriga must be fired FOR EACH ROW");
-
- if (current_context) {
- int i;
- for (i = 0; i < tg->tg_trigger->tgnargs; i++) {
- char *arg = tg->tg_trigger->tgargs[i];
- if (strcmp(arg, current_context) == 0)
- return PointerGetDatum(NULL);
- }
- }
-
- elog(ERROR, "action denied");
-}
-
diff --git a/sql/pgq/triggers/logtriga.c b/sql/pgq/triggers/logtriga.c
index d27cfb54..b724dd60 100644
--- a/sql/pgq/triggers/logtriga.c
+++ b/sql/pgq/triggers/logtriga.c
@@ -56,6 +56,9 @@ pgq_logtriga(PG_FUNCTION_ARGS)
if (!TRIGGER_FIRED_AFTER(tg->tg_event))
elog(ERROR, "pgq.logtriga must be fired AFTER");
+ if (pgq_is_logging_disabled())
+ goto skip_it;
+
/*
* Connect to the SPI manager
*/
@@ -76,6 +79,7 @@ pgq_logtriga(PG_FUNCTION_ARGS)
if (SPI_finish() < 0)
elog(ERROR, "SPI_finish failed");
+skip_it:
return PointerGetDatum(NULL);
}
diff --git a/sql/pgq/triggers/logutriga.c b/sql/pgq/triggers/logutriga.c
index 17819c58..1dac75a5 100644
--- a/sql/pgq/triggers/logutriga.c
+++ b/sql/pgq/triggers/logutriga.c
@@ -95,6 +95,8 @@ pgq_logutriga(PG_FUNCTION_ARGS)
else
row = tg->tg_trigtuple;
+ if (pgq_is_logging_disabled())
+ goto skip_it;
/*
* Connect to the SPI manager
@@ -126,6 +128,7 @@ pgq_logutriga(PG_FUNCTION_ARGS)
* After trigger ignores result,
* before trigger skips event if NULL.
*/
+skip_it:
if (TRIGGER_FIRED_AFTER(tg->tg_event) || ev.skip)
return PointerGetDatum(NULL);
else
diff --git a/sql/pgq/triggers/pgq_triggers.sql.in b/sql/pgq/triggers/pgq_triggers.sql.in
index 3edf1966..93f9ba7a 100644
--- a/sql/pgq/triggers/pgq_triggers.sql.in
+++ b/sql/pgq/triggers/pgq_triggers.sql.in
@@ -97,34 +97,3 @@ AS 'MODULE_PATHNAME', 'pgq_sqltriga' LANGUAGE C;
CREATE OR REPLACE FUNCTION pgq.logutriga() RETURNS TRIGGER
AS 'MODULE_PATHNAME', 'pgq_logutriga' LANGUAGE C;
--- ----------------------------------------------------------------------
--- Function: pgq.denytriga()
---
--- Trigger function that denies operation unless in particular context.
---
--- Purpose:
--- Used as producer for several PgQ standard consumers (cube_dispatcher,
--- queue_mover, table_dispatcher). Basically for cases where the
--- consumer wants to parse the event and look at the actual column values.
---
--- Trigger parameters:
--- One or more context names to allow.
---
--- Regular listen trigger example:
--- > CREATE TRIGGER triga_nimi AFTER INSERT OR UPDATE ON customer
--- > FOR EACH ROW EXECUTE PROCEDURE pgq.denytriga('my_replica');
--- ----------------------------------------------------------------------
-CREATE OR REPLACE FUNCTION pgq.denytriga() RETURNS TRIGGER
-AS 'MODULE_PATHNAME', 'pgq_denytriga' LANGUAGE C;
-
--- ----------------------------------------------------------------------
--- Function: pgq.set_connection_context(1)
---
--- This function sets the context for current connections.
---
--- Parameters:
--- context - context name
--- ----------------------------------------------------------------------
-CREATE OR REPLACE FUNCTION pgq.set_connection_context(i_context text) RETURNS void
-AS 'MODULE_PATHNAME', 'pgq_set_connection_context' LANGUAGE C;
-
diff --git a/sql/pgq/triggers/sqltriga.c b/sql/pgq/triggers/sqltriga.c
index 094cc319..6dbb1cd1 100644
--- a/sql/pgq/triggers/sqltriga.c
+++ b/sql/pgq/triggers/sqltriga.c
@@ -49,13 +49,17 @@ pgq_sqltriga(PG_FUNCTION_ARGS)
if (!CALLED_AS_TRIGGER(fcinfo))
elog(ERROR, "pgq.logutriga not called as trigger");
+ tg = (TriggerData *) (fcinfo->context);
+
+ if (pgq_is_logging_disabled())
+ goto skip_it;
+
/*
* Connect to the SPI manager
*/
if (SPI_connect() < 0)
elog(ERROR, "logtriga: SPI_connect() failed");
- tg = (TriggerData *) (fcinfo->context);
pgq_prepare_event(&ev, tg, true);
appendStringInfoChar(ev.ev_type, ev.op_type);
@@ -74,6 +78,7 @@ pgq_sqltriga(PG_FUNCTION_ARGS)
* After trigger ignores result,
* before trigger skips event if NULL.
*/
+skip_it:
if (TRIGGER_FIRED_AFTER(tg->tg_event) || ev.skip)
return PointerGetDatum(NULL);
else if (TRIGGER_FIRED_BY_UPDATE(tg->tg_event))