diff options
author | Marko Kreen | 2009-02-13 10:29:12 +0000 |
---|---|---|
committer | Marko Kreen | 2009-02-13 12:21:08 +0000 |
commit | 67e3ac04a87179e4283f71a5b8a55e3ddde30791 (patch) | |
tree | 8c8bb31d9ded22b010ffc89142eebabc15bf2a90 | |
parent | 5521e5fc2f399a923fa7fb313bbe797cfa0d5baa (diff) |
sql/pgq update
structure:
- event_seq field for pgq.tick table, to store last value
from event_id_seq
- 'disabled' field for pgq.queue, which can disable any
direct inserts to queue. inserts are allowed only
if session_role = 'replica'
functions:
- pgq.next_batch_info() as next_batch() but returns full details
- pgq.ticker(): now contains ticker logic that previously
was located in python code - it uses the event_seq field
in tick table to know about last sequence value
- pgq.insert_event_raw() - check disabled flag
- pgq.seq_getval() / pgq.seq_setval() functions for safe
sequence variable manipulation
- remove denytriga, now regular triggers can play the role
- use OUT parameters instead of ret types
- pgq.force_tick(): silently ignore the request on disabled queues
- pgq.maint_retry_events(): fwd port bugfix from 2.1-stable
- pgq.maint_rotate_tables(): cleanup from 2.1-stable
- pgq.maint_tables_to_vacuum(): add more tables
- triggers/common.c: bugfix from 2.1-stable
30 files changed, 586 insertions, 465 deletions
diff --git a/sql/pgq/Makefile b/sql/pgq/Makefile index 26c15426..37facee4 100644 --- a/sql/pgq/Makefile +++ b/sql/pgq/Makefile @@ -7,9 +7,9 @@ DATA = structure/uninstall_pgq.sql LDRS = structure/func_internal.sql structure/func_public.sql structure/triggers.sql FUNCS = $(shell sed -n -e '/^\\/{s/\\i //;p}' $(LDRS)) SRCS = structure/tables.sql structure/grants.sql structure/install.sql \ - structure/types.sql structure/uninstall_pgq.sql $(FUNCS) + structure/uninstall_pgq.sql $(FUNCS) -REGRESS = pgq_init pgq_core logutriga sqltriga denytriga +REGRESS = pgq_init pgq_core logutriga sqltriga REGRESS_OPTS = --load-language=plpgsql include ../../config.mak @@ -51,7 +51,7 @@ pgq.upgrade.sql: $(SRCS) dox: cleandox $(SRCS) mkdir -p docs/html mkdir -p docs/sql - $(CATSQL) --ndoc structure/tables.sql structure/types.sql > docs/sql/schema.sql + $(CATSQL) --ndoc structure/tables.sql > docs/sql/schema.sql $(CATSQL) --ndoc structure/func_public.sql > docs/sql/external.sql $(CATSQL) --ndoc structure/func_internal.sql > docs/sql/internal.sql $(CATSQL) --ndoc structure/triggers.sql > docs/sql/triggers.sql diff --git a/sql/pgq/expected/denytriga.out b/sql/pgq/expected/denytriga.out deleted file mode 100644 index 4dac73d4..00000000 --- a/sql/pgq/expected/denytriga.out +++ /dev/null @@ -1,37 +0,0 @@ -create table denytest ( - id integer -); -create trigger denytrg after insert or update or delete -on denytest for each row execute procedure pgq.denytriga('baz'); -insert into denytest values (1); -- must fail -ERROR: action denied -select pgq.set_connection_context('foo'); - set_connection_context ------------------------- - -(1 row) - -insert into denytest values (1); -- must fail -ERROR: action denied -select pgq.set_connection_context('baz'); - set_connection_context ------------------------- - -(1 row) - -insert into denytest values (1); -- must succeed -select pgq.set_connection_context(null); - set_connection_context ------------------------- - -(1 row) - -delete from denytest; -- must fail -ERROR: action denied -select pgq.set_connection_context('baz'); - set_connection_context ------------------------- - -(1 row) - -delete from denytest; -- must succeed diff --git a/sql/pgq/expected/pgq_core.out b/sql/pgq/expected/pgq_core.out index 21baed32..e90786ca 100644 --- a/sql/pgq/expected/pgq_core.out +++ b/sql/pgq/expected/pgq_core.out @@ -50,6 +50,7 @@ select pgq.register_consumer('myqueue', 'consumer'); 1 (1 row) +update pgq.queue set queue_ticker_max_lag = '0', queue_ticker_idle_period = '0'; select pgq.next_batch('myqueue', 'consumer'); next_batch ------------ @@ -80,10 +81,10 @@ select pgq.next_batch('myqueue', 'consumer'); 1 (1 row) -select queue_name, consumer_name, prev_tick_id, tick_id, lag from pgq.get_batch_info(1); - queue_name | consumer_name | prev_tick_id | tick_id | lag -------------+---------------+--------------+---------+------------- - myqueue | consumer | 1 | 2 | @ 0.00 secs +select queue_name, consumer_name, prev_tick_id, tick_id, lag < '30 seconds' as lag_exists from pgq.get_batch_info(1); + queue_name | consumer_name | prev_tick_id | tick_id | lag_exists +------------+---------------+--------------+---------+------------ + myqueue | consumer | 1 | 2 | t (1 row) select queue_name, queue_ntables, queue_cur_table, queue_rotation_period, @@ -93,7 +94,7 @@ select queue_name, queue_ntables, queue_cur_table, queue_rotation_period, from pgq.get_queue_info() order by 1; queue_name | queue_ntables | queue_cur_table | queue_rotation_period | switch_time_exists | queue_external_ticker | queue_ticker_max_count | queue_ticker_max_lag | queue_ticker_idle_period | ticker_lag_exists ------------+---------------+-----------------+-----------------------+--------------------+-----------------------+------------------------+----------------------+--------------------------+------------------- - myqueue | 3 | 0 | @ 2 hours | t | f | 500 | @ 3 secs | @ 1 min | t + myqueue | 3 | 0 | @ 2 hours | t | f | 500 | @ 0 | @ 0 | t (1 row) select queue_name, consumer_name, lag < '30 seconds' as lag_exists, @@ -290,6 +291,56 @@ select pgq.force_tick('myqueue'); select nextval(queue_event_seq) from pgq.queue where queue_name = 'myqueue'; nextval --------- - 1006 + 2006 +(1 row) + +create sequence tmptest_seq; +select pgq.seq_getval('tmptest_seq'); + seq_getval +------------ + 1 +(1 row) + +select pgq.seq_setval('tmptest_seq', 10); + seq_setval +------------ + 10 +(1 row) + +select pgq.seq_setval('tmptest_seq', 5); + seq_setval +------------ + 10 +(1 row) + +select pgq.seq_setval('tmptest_seq', 15); + seq_setval +------------ + 15 +(1 row) + +select pgq.seq_getval('tmptest_seq'); + seq_getval +------------ + 15 +(1 row) + +-- test disabled +select pgq.insert_event('myqueue', 'test', 'event'); + insert_event +-------------- + 2007 +(1 row) + +update pgq.queue set queue_disable_insert = true where queue_name = 'myqueue'; +select pgq.insert_event('myqueue', 'test', 'event'); +ERROR: Insert into queue disallowed +CONTEXT: PL/pgSQL function "insert_event" line 19 at RETURN +PL/pgSQL function "insert_event" line 15 at RETURN +update pgq.queue set queue_disable_insert = false where queue_name = 'myqueue'; +select pgq.insert_event('myqueue', 'test', 'event'); + insert_event +-------------- + 2009 (1 row) diff --git a/sql/pgq/functions/pgq.current_event_table.sql b/sql/pgq/functions/pgq.current_event_table.sql index 5349bb7e..3e4d74fb 100644 --- a/sql/pgq/functions/pgq.current_event_table.sql +++ b/sql/pgq/functions/pgq.current_event_table.sql @@ -18,12 +18,20 @@ returns text as $$ -- ---------------------------------------------------------------------- declare res text; + disabled boolean; begin - select queue_data_pfx || '_' || queue_cur_table into res + select queue_data_pfx || '_' || queue_cur_table, + queue_disable_insert + into res, disabled from pgq.queue where queue_name = x_queue_name; if not found then raise exception 'Event queue not found'; end if; + if disabled then + if current_setting('session_replication_role') <> 'replica' then + raise exception 'Writing to queue disabled'; + end if; + end if; return res; end; $$ language plpgsql; -- no perms needed diff --git a/sql/pgq/functions/pgq.force_tick.sql b/sql/pgq/functions/pgq.force_tick.sql index 0ea098d9..c1550d9a 100644 --- a/sql/pgq/functions/pgq.force_tick.sql +++ b/sql/pgq/functions/pgq.force_tick.sql @@ -27,13 +27,14 @@ begin -- bump seq and get queue id select queue_id, setval(queue_event_seq, nextval(queue_event_seq) - + queue_ticker_max_count * 2) as tmp + + queue_ticker_max_count * 2 + 1000) as tmp into q from pgq.queue where queue_name = i_queue_name and not queue_external_ticker; - if not found then - raise exception 'queue not found or ticks not allowed'; - end if; + + --if not found then + -- raise notice 'queue not found or ticks not allowed'; + --end if; -- return last tick id select tick_id into t from pgq.tick diff --git a/sql/pgq/functions/pgq.get_batch_events.sql b/sql/pgq/functions/pgq.get_batch_events.sql index 8166d519..3fadaaf7 100644 --- a/sql/pgq/functions/pgq.get_batch_events.sql +++ b/sql/pgq/functions/pgq.get_batch_events.sql @@ -1,5 +1,16 @@ -create or replace function pgq.get_batch_events(x_batch_id bigint) -returns setof pgq.ret_batch_event as $$ +create or replace function pgq.get_batch_events( + in x_batch_id bigint, + out ev_id bigint, + out ev_time timestamptz, + out ev_txid bigint, + out ev_retry int4, + out ev_type text, + out ev_data text, + out ev_extra1 text, + out ev_extra2 text, + out ev_extra3 text, + out ev_extra4 text) +returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: pgq.get_batch_events(1) -- @@ -11,16 +22,18 @@ returns setof pgq.ret_batch_event as $$ -- Returns: -- List of events. -- ---------------------------------------------------------------------- -declare - rec pgq.ret_batch_event%rowtype; - sql text; -begin - sql := pgq.batch_event_sql(x_batch_id); - for rec in execute sql loop - return next rec; - end loop; +declare + sql text; +begin + sql := pgq.batch_event_sql(x_batch_id); + for ev_id, ev_time, ev_txid, ev_retry, ev_type, ev_data, + ev_extra1, ev_extra2, ev_extra3, ev_extra4 + in execute sql + loop + return next; + end loop; return; -end; +end; $$ language plpgsql; -- no perms needed diff --git a/sql/pgq/functions/pgq.get_consumer_info.sql b/sql/pgq/functions/pgq.get_consumer_info.sql index 27dd444d..a101c9b3 100644 --- a/sql/pgq/functions/pgq.get_consumer_info.sql +++ b/sql/pgq/functions/pgq.get_consumer_info.sql @@ -1,7 +1,13 @@ -------------------------------------------------------------------------- -create or replace function pgq.get_consumer_info() -returns setof pgq.ret_consumer_info as $$ +create or replace function pgq.get_consumer_info( + out queue_name text, + out consumer_name text, + out lag interval, + out last_seen interval, + out last_tick bigint, + out current_batch bigint, + out next_tick bigint) +returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: pgq.get_consumer_info(0) -- @@ -10,73 +16,75 @@ returns setof pgq.ret_consumer_info as $$ -- Returns: -- See pgq.get_consumer_info(2) -- ---------------------------------------------------------------------- -declare - ret pgq.ret_consumer_info%rowtype; - i record; begin - for i in select queue_name from pgq.queue order by 1 + for queue_name, consumer_name, lag, last_seen, + last_tick, current_batch, next_tick + in + select f.queue_name, f.consumer_name, f.lag, f.last_seen, + f.last_tick, f.current_batch, f.next_tick + from pgq.get_consumer_info(null, null) f loop - for ret in - select * from pgq.get_consumer_info(i.queue_name) - loop - return next ret; - end loop; + return next; end loop; return; end; $$ language plpgsql security definer; -------------------------------------------------------------------------- -create or replace function pgq.get_consumer_info(x_queue_name text) -returns setof pgq.ret_consumer_info as $$ + +create or replace function pgq.get_consumer_info( + in i_queue_name text, + out queue_name text, + out consumer_name text, + out lag interval, + out last_seen interval, + out last_tick bigint, + out current_batch bigint, + out next_tick bigint) +returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: pgq.get_consumer_info(1) -- --- Returns info about consumers on one particular queue. --- --- Parameters: --- x_queue_name - Queue name +-- Returns info about all consumers on single queue. -- -- Returns: -- See pgq.get_consumer_info(2) -- ---------------------------------------------------------------------- -declare - ret pgq.ret_consumer_info%rowtype; - tmp record; begin - for tmp in - select queue_name, co_name - from pgq.queue, pgq.consumer, pgq.subscription - where queue_id = sub_queue - and co_id = sub_consumer - and queue_name = x_queue_name - order by 1, 2 + for queue_name, consumer_name, lag, last_seen, + last_tick, current_batch, next_tick + in + select f.queue_name, f.consumer_name, f.lag, f.last_seen, + f.last_tick, f.current_batch, f.next_tick + from pgq.get_consumer_info(i_queue_name, null) f loop - for ret in - select * from pgq.get_consumer_info(tmp.queue_name, tmp.co_name) - loop - return next ret; - end loop; + return next; end loop; return; end; $$ language plpgsql security definer; ------------------------------------------------------------------------- + create or replace function pgq.get_consumer_info( - x_queue_name text, - x_consumer_name text) -returns setof pgq.ret_consumer_info as $$ + in i_queue_name text, + in i_consumer_name text, + out queue_name text, + out consumer_name text, + out lag interval, + out last_seen interval, + out last_tick bigint, + out current_batch bigint, + out next_tick bigint) +returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: pgq.get_consumer_info(2) -- -- Get info about particular consumer on particular queue. -- -- Parameters: --- x_queue_name - name of a queue. --- x_consumer_name - name of a consumer +-- i_queue_name - name of a queue. (null = all) +-- i_consumer_name - name of a consumer (null = all) -- -- Returns: -- queue_name - Queue name @@ -87,26 +95,24 @@ returns setof pgq.ret_consumer_info as $$ -- current_batch - Current batch ID, if one is active or NULL -- next_tick - If batch is active, then its final tick. -- ---------------------------------------------------------------------- -declare - ret pgq.ret_consumer_info%rowtype; begin - for ret in - select queue_name, co_name, - current_timestamp - tick_time as lag, - current_timestamp - sub_active as last_seen, - sub_last_tick as last_tick, - sub_batch as current_batch, - sub_next_tick as next_tick - from pgq.subscription, pgq.tick, pgq.queue, pgq.consumer - where tick_id = sub_last_tick - and queue_id = sub_queue - and tick_queue = sub_queue - and co_id = sub_consumer - and queue_name = x_queue_name - and co_name = x_consumer_name + for queue_name, consumer_name, lag, last_seen, + last_tick, current_batch, next_tick + in + select q.queue_name, c.co_name, + current_timestamp - t.tick_time, + current_timestamp - s.sub_active, + s.sub_last_tick, s.sub_batch, s.sub_next_tick + from pgq.subscription s, pgq.tick t, pgq.queue q, pgq.consumer c + where t.tick_id = s.sub_last_tick + and q.queue_id = s.sub_queue + and t.tick_queue = s.sub_queue + and c.co_id = s.sub_consumer + and (i_queue_name is null or q.queue_name = i_queue_name) + and (i_consumer_name is null or c.co_name = i_consumer_name) order by 1,2 loop - return next ret; + return next; end loop; return; end; diff --git a/sql/pgq/functions/pgq.get_queue_info.sql b/sql/pgq/functions/pgq.get_queue_info.sql index 9d15d49e..14906770 100644 --- a/sql/pgq/functions/pgq.get_queue_info.sql +++ b/sql/pgq/functions/pgq.get_queue_info.sql @@ -1,5 +1,15 @@ -create or replace function pgq.get_queue_info() -returns setof pgq.ret_queue_info as $$ +create or replace function pgq.get_queue_info( + out queue_name text, + out queue_ntables integer, + out queue_cur_table integer, + out queue_rotation_period interval, + out queue_switch_time timestamptz, + out queue_external_ticker boolean, + out queue_ticker_max_count integer, + out queue_ticker_max_lag interval, + out queue_ticker_idle_period interval, + out ticker_lag interval) +returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: pgq.get_queue_info(0) -- @@ -8,22 +18,35 @@ returns setof pgq.ret_queue_info as $$ -- Returns: -- List of pgq.ret_queue_info records. -- ---------------------------------------------------------------------- -declare - q record; - ret pgq.ret_queue_info%rowtype; begin - for q in - select queue_name from pgq.queue order by 1 + for queue_name, queue_ntables, queue_cur_table, queue_rotation_period, + queue_switch_time, queue_external_ticker, queue_ticker_max_count, + queue_ticker_max_lag, queue_ticker_idle_period, ticker_lag + in select + f.queue_name, f.queue_ntables, f.queue_cur_table, f.queue_rotation_period, + f.queue_switch_time, f.queue_external_ticker, f.queue_ticker_max_count, + f.queue_ticker_max_lag, f.queue_ticker_idle_period, f.ticker_lag + from pgq.get_queue_info(null) f loop - select * into ret from pgq.get_queue_info(q.queue_name); - return next ret; + return next; end loop; return; end; -$$ language plpgsql security definer; +$$ language plpgsql; -create or replace function pgq.get_queue_info(qname text) -returns pgq.ret_queue_info as $$ +create or replace function pgq.get_queue_info( + in i_queue_name text, + out queue_name text, + out queue_ntables integer, + out queue_cur_table integer, + out queue_rotation_period interval, + out queue_switch_time timestamptz, + out queue_external_ticker boolean, + out queue_ticker_max_count integer, + out queue_ticker_max_lag interval, + out queue_ticker_idle_period interval, + out ticker_lag interval) +returns setof record as $$ -- ---------------------------------------------------------------------- -- Function: pgq.get_queue_info(1) -- @@ -32,20 +55,26 @@ returns pgq.ret_queue_info as $$ -- Returns: -- One pgq.ret_queue_info record. -- ---------------------------------------------------------------------- -declare - ret pgq.ret_queue_info%rowtype; begin - select queue_name, queue_ntables, queue_cur_table, - queue_rotation_period, queue_switch_time, - queue_external_ticker, - queue_ticker_max_count, queue_ticker_max_lag, - queue_ticker_idle_period, - (select current_timestamp - tick_time - from pgq.tick where tick_queue = queue_id - order by tick_queue desc, tick_id desc limit 1 - ) as ticker_lag - into ret from pgq.queue where queue_name = qname; - return ret; + for queue_name, queue_ntables, queue_cur_table, queue_rotation_period, + queue_switch_time, queue_external_ticker, queue_ticker_max_count, + queue_ticker_max_lag, queue_ticker_idle_period, ticker_lag + in select + q.queue_name, q.queue_ntables, q.queue_cur_table, + q.queue_rotation_period, q.queue_switch_time, + q.queue_external_ticker, + q.queue_ticker_max_count, q.queue_ticker_max_lag, + q.queue_ticker_idle_period, + (select current_timestamp - tick_time + from pgq.tick where tick_queue = queue_id + order by tick_queue desc, tick_id desc limit 1) + from pgq.queue q + where (i_queue_name is null or q.queue_name = i_queue_name) + order by q.queue_name + loop + return next; + end loop; + return; end; -$$ language plpgsql security definer; +$$ language plpgsql; diff --git a/sql/pgq/functions/pgq.maint_retry_events.sql b/sql/pgq/functions/pgq.maint_retry_events.sql index f3038b86..8e4bfa4b 100644 --- a/sql/pgq/functions/pgq.maint_retry_events.sql +++ b/sql/pgq/functions/pgq.maint_retry_events.sql @@ -8,9 +8,6 @@ returns integer as $$ -- It moves small amount at a time. It should be called -- until it returns 0 -- --- Parameters: --- arg - desc --- -- Returns: -- Number of events processed. -- ---------------------------------------------------------------------- @@ -20,10 +17,9 @@ declare begin cnt := 0; for rec in - select pgq.insert_event_raw(queue_name, - ev_id, ev_time, ev_owner, ev_retry, ev_type, ev_data, - ev_extra1, ev_extra2, ev_extra3, ev_extra4), - ev_owner, ev_id + select queue_name, + ev_id, ev_time, ev_owner, ev_retry, ev_type, ev_data, + ev_extra1, ev_extra2, ev_extra3, ev_extra4 from pgq.retry_queue, pgq.queue, pgq.subscription where ev_retry_after <= current_timestamp and sub_id = ev_owner @@ -32,6 +28,10 @@ begin limit 10 loop cnt := cnt + 1; + perform pgq.insert_event_raw(rec.queue_name, + rec.ev_id, rec.ev_time, rec.ev_owner, rec.ev_retry, + rec.ev_type, rec.ev_data, rec.ev_extra1, rec.ev_extra2, + rec.ev_extra3, rec.ev_extra4); delete from pgq.retry_queue where ev_owner = rec.ev_owner and ev_id = rec.ev_id; diff --git a/sql/pgq/functions/pgq.maint_rotate_tables.sql b/sql/pgq/functions/pgq.maint_rotate_tables.sql index b52a5025..d24f09c8 100644 --- a/sql/pgq/functions/pgq.maint_rotate_tables.sql +++ b/sql/pgq/functions/pgq.maint_rotate_tables.sql @@ -9,13 +9,15 @@ returns integer as $$ -- i_queue_name - Name of the queue -- -- Returns: --- nothing +-- 1 if rotation happened, otherwise 0. -- ---------------------------------------------------------------------- declare - badcnt integer; - cf record; - nr integer; - tbl text; + badcnt integer; + cf record; + nr integer; + tbl text; + lowest_tick_id int8; + lowest_xmin int8; begin -- check if needed and load record select * from pgq.queue into cf @@ -28,22 +30,26 @@ begin return 0; end if; - -- check if any consumer is on previous table - select coalesce(count(*), 0) into badcnt - from pgq.subscription, pgq.tick - where txid_snapshot_xmin(tick_snapshot) < cf.queue_switch_step2 - and sub_queue = cf.queue_id - and tick_queue = cf.queue_id - and tick_id = (select tick_id from pgq.tick - where tick_id < sub_last_tick - and tick_queue = sub_queue - order by tick_queue desc, tick_id desc - limit 1); - if badcnt > 0 then - return 0; + -- find lowest tick for that queue + select min(sub_last_tick) into lowest_tick_id + from pgq.subscription + where sub_queue = cf.queue_id; + + -- if some consumer exists + if lowest_tick_id is not null then + -- is the slowest one still on previous table? + select txid_snapshot_xmin(tick_snapshot) into lowest_xmin + from pgq.tick + where tick_queue = cf.queue_id + and tick_id = lowest_tick_id; + if lowest_xmin <= cf.queue_switch_step2 then + return 0; -- skip rotation then + end if; end if; - -- all is fine, calc next table number + -- nobody on previous table, we can rotate + + -- calc next table number and name nr := cf.queue_cur_table + 1; if nr = cf.queue_ntables then nr := 0; @@ -57,7 +63,7 @@ begin execute 'truncate ' || tbl; exception when lock_not_available then - raise warning 'truncate of % failed, skipping rotate', tbl; + -- cannot truncate, skipping rotate return 0; end; @@ -69,7 +75,14 @@ begin queue_switch_step2 = NULL where queue_id = cf.queue_id; - -- clean ticks - avoid partial batches + -- Clean ticks by using step2 txid from previous rotation. + -- That should keep all ticks for all batches that are completely + -- in old table. This keeps them for longer than needed, but: + -- 1. we want the pgq.tick table to be big, to avoid Postgres + -- accitentally switching to seqscans on that. + -- 2. that way we guarantee to consumers that they an be moved + -- back on the queue at least for one rotation_period. + -- (may help in disaster recovery) delete from pgq.tick where tick_queue = cf.queue_id and txid_snapshot_xmin(tick_snapshot) < cf.queue_switch_step2; @@ -78,16 +91,15 @@ begin end; $$ language plpgsql; -- need admin access + +create or replace function pgq.maint_rotate_tables_step2() +returns integer as $$ -- ---------------------------------------------------------------------- -- Function: pgq.maint_rotate_tables_step2(0) -- --- It tag rotation as finished where needed. It should be +-- Stores the txid when the rotation was visible. It should be -- called in separate transaction than pgq.maint_rotate_tables_step1() -- ---------------------------------------------------------------------- -create or replace function pgq.maint_rotate_tables_step2() -returns integer as $$ --- visibility tracking. this should run in separate --- tranaction than step1 begin update pgq.queue set queue_switch_step2 = txid_current() diff --git a/sql/pgq/functions/pgq.maint_tables_to_vacuum.sql b/sql/pgq/functions/pgq.maint_tables_to_vacuum.sql index b2469d0c..deebb356 100644 --- a/sql/pgq/functions/pgq.maint_tables_to_vacuum.sql +++ b/sql/pgq/functions/pgq.maint_tables_to_vacuum.sql @@ -11,35 +11,40 @@ returns setof text as $$ -- List of table names. -- ---------------------------------------------------------------------- declare - row record; + scm text; + tbl text; + fqname text; begin - return next 'pgq.subscription'; - return next 'pgq.consumer'; - return next 'pgq.queue'; - return next 'pgq.tick'; - return next 'pgq.retry_queue'; - - -- include also txid, pgq_ext and londiste tables if they exist - for row in - select n.nspname as scm, t.relname as tbl - from pg_class t, pg_namespace n - where n.oid = t.relnamespace - and n.nspname = 'txid' and t.relname = 'epoch' - union all - select n.nspname as scm, t.relname as tbl - from pg_class t, pg_namespace n - where n.oid = t.relnamespace - and n.nspname = 'londiste' and t.relname = 'completed' - union all - select n.nspname as scm, t.relname as tbl - from pg_class t, pg_namespace n - where n.oid = t.relnamespace - and n.nspname = 'pgq_ext' - and t.relname in ('completed_tick', 'completed_batch', 'completed_event', 'partial_batch') + for scm, tbl in values + ('pgq', 'subscription'), + ('pgq', 'consumer'), + ('pgq', 'queue'), + ('pgq', 'tick'), + ('pgq', 'retry_queue'), + ('pgq_ext', 'completed_tick'), + ('pgq_ext', 'completed_batch'), + ('pgq_ext', 'completed_event'), + ('pgq_ext', 'partial_batch'), + --('pgq_node', 'node_location'), + --('pgq_node', 'node_info'), + ('pgq_node', 'local_state'), + --('pgq_node', 'subscriber_info'), + --('londiste', 'table_info'), + ('londiste', 'seq_info'), + --('londiste', 'applied_execute'), + --('londiste', 'pending_fkeys'), + ('txid', 'epoch'), + ('londiste', 'completed') loop - return next row.scm || '.' || row.tbl; + select n.nspname || '.' || t.relname into fqname + from pg_class t, pg_namespace n + where n.oid = t.relnamespace + and n.nspname = scm + and t.relname = tbl; + if found then + return next fqname; + end if; end loop; - return; end; $$ language plpgsql; diff --git a/sql/pgq/functions/pgq.next_batch.sql b/sql/pgq/functions/pgq.next_batch.sql index 88a40e4d..5f9981b2 100644 --- a/sql/pgq/functions/pgq.next_batch.sql +++ b/sql/pgq/functions/pgq.next_batch.sql @@ -1,67 +1,129 @@ -create or replace function pgq.next_batch(x_queue_name text, x_consumer_name text) -returns bigint as $$ +create or replace function pgq.next_batch_info( + in i_queue_name text, + in i_consumer_name text, + out batch_id int8, + out cur_tick_id int8, + out prev_tick_id int8, + out cur_tick_time timestamptz, + out prev_tick_time timestamptz, + out cur_tick_event_seq int8, + out prev_tick_event_seq int8) +as $$ -- ---------------------------------------------------------------------- --- Function: pgq.next_batch(2) +-- Function: pgq.next_batch_info(2) -- -- Makes next block of events active. -- -- If it returns NULL, there is no events available in queue. --- Consumer should sleep a bith then. +-- Consumer should sleep then. +-- +-- The values from event_id sequence may give hint how big the +-- batch may be. But they are inexact, they do not give exact size. +-- Client *MUST NOT* use them to detect whether the batch contains any +-- events at all - the values are unfit for that purpose. -- -- Parameters: --- x_queue_name - Name of the queue --- x_consumer_name - Name of the consumer +-- i_queue_name - Name of the queue +-- i_consumer_name - Name of the consumer -- -- Returns: --- Batch ID or NULL if there are no more events available. +-- batch_id - Batch ID or NULL if there are no more events available. +-- cur_tick_id - End tick id. +-- cur_tick_time - End tick time. +-- cur_tick_event_seq - Value from event id sequence at the time tick was issued. +-- prev_tick_id - Start tick id. +-- prev_tick_time - Start tick time. +-- prev_tick_event_seq - value from event id sequence at the time tick was issued. -- ---------------------------------------------------------------------- declare - next_tick bigint; - batch_id bigint; errmsg text; - sub record; + queue_id integer; + sub_id integer; + cons_id integer; begin - select sub_queue, sub_consumer, sub_id, sub_last_tick, sub_batch into sub - from pgq.queue q, pgq.consumer c, pgq.subscription s - where q.queue_name = x_queue_name - and c.co_name = x_consumer_name + select s.sub_queue, s.sub_consumer, s.sub_id, s.sub_batch, + t1.tick_id, t1.tick_time, t1.tick_event_seq, + t2.tick_id, t2.tick_time, t2.tick_event_seq + into queue_id, cons_id, sub_id, batch_id, + prev_tick_id, prev_tick_time, prev_tick_event_seq, + cur_tick_id, cur_tick_time, cur_tick_event_seq + from pgq.consumer c, + pgq.queue q, + pgq.subscription s + left join pgq.tick t1 + on (t1.tick_queue = s.sub_queue + and t1.tick_id = s.sub_last_tick) + left join pgq.tick t2 + on (t2.tick_queue = s.sub_queue + and t2.tick_id = s.sub_next_tick) + where q.queue_name = i_queue_name + and c.co_name = i_consumer_name and s.sub_queue = q.queue_id and s.sub_consumer = c.co_id; if not found then errmsg := 'Not subscriber to queue: ' - || coalesce(x_queue_name, 'NULL') + || coalesce(i_queue_name, 'NULL') || '/' - || coalesce(x_consumer_name, 'NULL'); + || coalesce(i_consumer_name, 'NULL'); raise exception '%', errmsg; end if; -- has already active batch - if sub.sub_batch is not null then - return sub.sub_batch; + if batch_id is not null then + return; end if; -- find next tick - select tick_id into next_tick + select tick_id, tick_time, tick_event_seq + into cur_tick_id, cur_tick_time, cur_tick_event_seq from pgq.tick - where tick_id > sub.sub_last_tick - and tick_queue = sub.sub_queue + where tick_id > prev_tick_id + and tick_queue = queue_id order by tick_queue asc, tick_id asc limit 1; if not found then -- nothing to do - return null; + prev_tick_id := null; + prev_tick_time := null; + prev_tick_event_seq := null; + return; end if; -- get next batch batch_id := nextval('pgq.batch_id_seq'); update pgq.subscription set sub_batch = batch_id, - sub_next_tick = next_tick, + sub_next_tick = cur_tick_id, sub_active = now() - where sub_queue = sub.sub_queue - and sub_consumer = sub.sub_consumer; - return batch_id; + where sub_queue = queue_id + and sub_consumer = cons_id; + return; end; $$ language plpgsql security definer; +create or replace function pgq.next_batch( + in i_queue_name text, + in i_consumer_name text) +returns int8 as $$ +-- ---------------------------------------------------------------------- +-- Function: pgq.next_batch(2) +-- +-- Old function that returns just batch_id. +-- +-- Parameters: +-- i_queue_name - Name of the queue +-- i_consumer_name - Name of the consumer +-- +-- Returns: +-- Batch ID or NULL if there are no more events available. +-- ---------------------------------------------------------------------- +declare + res int8; +begin + select batch_id into res + from pgq.next_batch_info(i_queue_name, i_consumer_name); + return res; +end; +$$ language plpgsql; + diff --git a/sql/pgq/functions/pgq.seq_funcs.sql b/sql/pgq/functions/pgq.seq_funcs.sql index 985f6389..4a31f3a9 100644 --- a/sql/pgq/functions/pgq.seq_funcs.sql +++ b/sql/pgq/functions/pgq.seq_funcs.sql @@ -4,7 +4,7 @@ returns bigint as $$ -- ---------------------------------------------------------------------- -- Function: pgq.seq_getval(1) -- --- read current last_val from seq, without offecting it. +-- Read current last_val from seq, without affecting it. -- -- Parameters: -- i_seq_name - Name of the sequence @@ -13,9 +13,23 @@ returns bigint as $$ -- last value. -- ---------------------------------------------------------------------- declare - res int8; + res int8; + fqname text; + pos integer; + s text; + n text; begin - execute 'select last_value from ' || i_seq_name into res; + pos := position('.' in i_seq_name); + if pos > 0 then + s := substring(i_seq_name for pos - 1); + n := substring(i_seq_name from pos + 1); + else + s := 'public'; + n := i_seq_name; + end if; + fqname := quote_ident(s) || '.' || quote_ident(n); + + execute 'select last_value from ' || fqname into res; return res; end; $$ language plpgsql; @@ -35,11 +49,25 @@ returns bigint as $$ -- current last value. -- ---------------------------------------------------------------------- declare - res int8; + res int8; + fqname text; + pos integer; + s text; + n text; begin + pos := position('.' in i_seq_name); + if pos > 0 then + s := substring(i_seq_name for pos - 1); + n := substring(i_seq_name from pos + 1); + else + s := 'public'; + n := i_seq_name; + end if; + fqname := quote_ident(s) || '.' || quote_ident(n); + res := pgq.seq_getval(i_seq_name); if res < i_new_value then - perform setval(i_seq_name, i_new_value); + perform setval(fqname, i_new_value); return i_new_value; end if; return res; diff --git a/sql/pgq/functions/pgq.ticker.sql b/sql/pgq/functions/pgq.ticker.sql index 598f43ec..ebfc907c 100644 --- a/sql/pgq/functions/pgq.ticker.sql +++ b/sql/pgq/functions/pgq.ticker.sql @@ -1,4 +1,4 @@ -create or replace function pgq.ticker(i_queue_name text, i_tick_id bigint, i_orig_timestamp timestamptz) +create or replace function pgq.ticker(i_queue_name text, i_tick_id bigint, i_orig_timestamp timestamptz, i_event_seq bigint) returns bigint as $$ -- ---------------------------------------------------------------------- -- Function: pgq.ticker(3) @@ -13,43 +13,32 @@ returns bigint as $$ -- Tick id. -- ---------------------------------------------------------------------- begin - insert into pgq.tick (tick_queue, tick_id, tick_time) - select queue_id, i_tick_id, i_orig_timestamp + insert into pgq.tick (tick_queue, tick_id, tick_time, tick_event_seq) + select queue_id, i_tick_id, i_orig_timestamp, i_event_seq from pgq.queue where queue_name = i_queue_name and queue_external_ticker; if not found then raise exception 'queue not found or external ticker disabled: %', i_queue_name; end if; + + -- make sure seqs stay current + perform pgq.seq_setval(queue_tick_seq, i_tick_id), + pgq.seq_setval(queue_event_seq, i_event_seq) + from pgq.queue + where queue_name = i_queue_name; + return i_tick_id; end; $$ language plpgsql security definer; -- unsure about access -create or replace function pgq.ticker(i_queue_name text, i_tick_id bigint) -returns bigint as $$ --- ---------------------------------------------------------------------- --- Function: pgq.ticker(2) --- --- External ticker: insert a tick with a particular tick_id. --- --- Parameters: --- i_queue_name - Name of the queue --- i_tick_id - Id of new tick. --- --- Returns: --- Tick id. --- ---------------------------------------------------------------------- -begin - return pgq.ticker(i_queue_name, i_tick_id, now()); -end; -$$ language plpgsql security definer; -- unsure about access create or replace function pgq.ticker(i_queue_name text) returns bigint as $$ -- ---------------------------------------------------------------------- -- Function: pgq.ticker(1) -- --- Insert a tick with a tick_id from sequence. +-- Check if tick is needed for the queue and insert it. -- -- For pgqadm usage. -- @@ -57,15 +46,19 @@ returns bigint as $$ -- i_queue_name - Name of the queue -- -- Returns: --- Tick id. +-- Tick id or NULL if no tick was done. -- ---------------------------------------------------------------------- declare res bigint; - ext boolean; - seq text; q record; + state record; + last2 record; begin - select queue_id, queue_tick_seq, queue_external_ticker into q + select queue_id, queue_tick_seq, queue_external_ticker, + queue_ticker_max_count, queue_ticker_max_lag, + queue_ticker_idle_period, queue_event_seq, + pgq.seq_getval(queue_event_seq) as event_seq + into q from pgq.queue where queue_name = i_queue_name; if not found then raise exception 'no such queue'; @@ -75,11 +68,51 @@ begin raise exception 'This queue has external tick source.'; end if; - insert into pgq.tick (tick_queue, tick_id) - values (q.queue_id, nextval(q.queue_tick_seq)); + -- load state from last tick + select now() - tick_time as lag, + q.event_seq - tick_event_seq as new_events, + tick_id, tick_time + into state + from pgq.tick + where tick_queue = q.queue_id + order by tick_queue desc, tick_id desc + limit 1; - res = currval(q.queue_tick_seq); - return res; + if found then + if state.new_events > 0 then + -- there are new events, should we wait a bit? + if state.new_events < q.queue_ticker_max_count + and state.lag < q.queue_ticker_max_lag + then + return NULL; + end if; + else + -- no new events, should we apply idle period? + -- check previous event from the last one. + select state.tick_time - tick_time as lag + into last2 + from pgq.tick + where tick_queue = q.queue_id + and tick_id < state.tick_id + order by tick_queue desc, tick_id desc + limit 1; + if found then + -- gradually decrease the tick frequency + if (state.lag < q.queue_ticker_max_lag / 2) + or + (state.lag < last2.lag * 2 + and state.lag < q.queue_ticker_idle_period) + then + return NULL; + end if; + end if; + end if; + end if; + + insert into pgq.tick (tick_queue, tick_id, tick_event_seq) + values (q.queue_id, nextval(q.queue_tick_seq), q.event_seq); + + return currval(q.queue_tick_seq); end; $$ language plpgsql security definer; -- unsure about access @@ -94,9 +127,18 @@ create or replace function pgq.ticker() returns bigint as $$ -- ---------------------------------------------------------------------- declare res bigint; + q record; begin - select count(pgq.ticker(queue_name)) into res - from pgq.queue where not queue_external_ticker; + res := 0; + for q in + select queue_name from pgq.queue + where not queue_external_ticker + order by queue_name + loop + if pgq.ticker(q.queue_name) > 0 then + res := res + 1; + end if; + end loop; return res; end; $$ language plpgsql security definer; diff --git a/sql/pgq/lowlevel/insert_event.c b/sql/pgq/lowlevel/insert_event.c index 0efbba74..5431c0cc 100644 --- a/sql/pgq/lowlevel/insert_event.c +++ b/sql/pgq/lowlevel/insert_event.c @@ -46,12 +46,14 @@ PG_FUNCTION_INFO_V1(pgq_insert_event_raw); */ #define QUEUE_SQL \ "select queue_id::int4, queue_data_pfx::text," \ - " queue_cur_table::int4, nextval(queue_event_seq)::int8 " \ + " queue_cur_table::int4, nextval(queue_event_seq)::int8," \ + " queue_disable_insert::bool" \ " from pgq.queue where queue_name = $1" #define COL_QUEUE_ID 1 #define COL_PREFIX 2 #define COL_TBLNO 3 #define COL_EVENT_ID 4 +#define COL_DISABLED 5 /* * Plan cache entry in HTAB. @@ -70,6 +72,7 @@ struct QueueState { int cur_table; char *table_prefix; Datum next_event_id; + bool disabled; }; /* @@ -193,6 +196,9 @@ static void load_queue_info(Datum queue_name, struct QueueState *state) state->next_event_id = SPI_getbinval(row, desc, COL_EVENT_ID, &isnull); if (isnull) elog(ERROR, "Seq name NULL"); + state->disabled = SPI_getbinval(row, desc, COL_DISABLED, &isnull); + if (isnull) + elog(ERROR, "insert_disabled NULL"); } /* @@ -221,7 +227,7 @@ pgq_insert_event_raw(PG_FUNCTION_ARGS) int i, res; if (PG_NARGS() < 6) - elog(ERROR, "too few args"); + elog(ERROR, "Need at least 6 arguments"); if (PG_ARGISNULL(0)) elog(ERROR, "Queue name must not be NULL"); @@ -232,6 +238,9 @@ pgq_insert_event_raw(PG_FUNCTION_ARGS) load_queue_info(PG_GETARG_DATUM(0), &state); + if (state.disabled) + elog(ERROR, "Insert into queue disallowed"); + if (PG_ARGISNULL(1)) ev_id = state.next_event_id; else diff --git a/sql/pgq/sql/denytriga.sql b/sql/pgq/sql/denytriga.sql deleted file mode 100644 index b2cc3261..00000000 --- a/sql/pgq/sql/denytriga.sql +++ /dev/null @@ -1,18 +0,0 @@ - -create table denytest ( - id integer -); - -create trigger denytrg after insert or update or delete -on denytest for each row execute procedure pgq.denytriga('baz'); - -insert into denytest values (1); -- must fail -select pgq.set_connection_context('foo'); -insert into denytest values (1); -- must fail -select pgq.set_connection_context('baz'); -insert into denytest values (1); -- must succeed -select pgq.set_connection_context(null); -delete from denytest; -- must fail -select pgq.set_connection_context('baz'); -delete from denytest; -- must succeed - diff --git a/sql/pgq/sql/pgq_core.sql b/sql/pgq/sql/pgq_core.sql index 247b4a64..1d655fb6 100644 --- a/sql/pgq/sql/pgq_core.sql +++ b/sql/pgq/sql/pgq_core.sql @@ -9,13 +9,14 @@ select pgq.drop_queue('tmpqueue'); select pgq.create_queue('myqueue'); select pgq.register_consumer('myqueue', 'consumer'); +update pgq.queue set queue_ticker_max_lag = '0', queue_ticker_idle_period = '0'; select pgq.next_batch('myqueue', 'consumer'); select pgq.next_batch('myqueue', 'consumer'); select pgq.ticker(); select pgq.next_batch('myqueue', 'consumer'); select pgq.next_batch('myqueue', 'consumer'); -select queue_name, consumer_name, prev_tick_id, tick_id, lag from pgq.get_batch_info(1); +select queue_name, consumer_name, prev_tick_id, tick_id, lag < '30 seconds' as lag_exists from pgq.get_batch_info(1); select queue_name, queue_ntables, queue_cur_table, queue_rotation_period, queue_switch_time <= now() as switch_time_exists, @@ -85,4 +86,10 @@ select pgq.seq_setval('tmptest_seq', 5); select pgq.seq_setval('tmptest_seq', 15); select pgq.seq_getval('tmptest_seq'); +-- test disabled +select pgq.insert_event('myqueue', 'test', 'event'); +update pgq.queue set queue_disable_insert = true where queue_name = 'myqueue'; +select pgq.insert_event('myqueue', 'test', 'event'); +update pgq.queue set queue_disable_insert = false where queue_name = 'myqueue'; +select pgq.insert_event('myqueue', 'test', 'event'); diff --git a/sql/pgq/sql/pgq_init.sql b/sql/pgq/sql/pgq_init.sql index 38dd7918..d14cf6ef 100644 --- a/sql/pgq/sql/pgq_init.sql +++ b/sql/pgq/sql/pgq_init.sql @@ -1,7 +1,8 @@ \set ECHO none \i ../txid/txid.sql -\i pgq.sql +-- \i pgq.sql +\i structure/install.sql \set ECHO all diff --git a/sql/pgq/structure/install.sql b/sql/pgq/structure/install.sql index 0dbc85ed..511aaba7 100644 --- a/sql/pgq/structure/install.sql +++ b/sql/pgq/structure/install.sql @@ -1,7 +1,6 @@ \i structure/tables.sql \i structure/grants.sql -\i structure/types.sql \i structure/func_internal.sql \i structure/func_public.sql \i structure/triggers.sql diff --git a/sql/pgq/structure/tables.sql b/sql/pgq/structure/tables.sql index 57e749c1..43d27287 100644 --- a/sql/pgq/structure/tables.sql +++ b/sql/pgq/structure/tables.sql @@ -36,11 +36,11 @@ create schema pgq; -- co_name - consumer's id for external usage -- ---------------------------------------------------------------------- create table pgq.consumer ( - co_id serial, - co_name text not null default 'fooz', + co_id serial, + co_name text not null default 'fooz', - constraint consumer_pkey primary key (co_id), - constraint consumer_name_uq UNIQUE (co_name) + constraint consumer_pkey primary key (co_id), + constraint consumer_name_uq UNIQUE (co_name) ); @@ -59,6 +59,7 @@ create table pgq.consumer ( -- queue_switch_step2 - tx after rotation was committed -- queue_switch_time - time when switch happened -- queue_external_ticker - ticks come from some external sources +-- queue_disable_insert - disallow pgq.insert_event() -- queue_ticker_max_count - batch should not contain more events -- queue_ticker_max_lag - events should not age more -- queue_ticker_idle_period - how often to tick when no events happen @@ -67,8 +68,8 @@ create table pgq.consumer ( -- queue_tick_seq - sequence for tick id's -- ---------------------------------------------------------------------- create table pgq.queue ( - queue_id serial, - queue_name text not null, + queue_id serial, + queue_name text not null, queue_ntables integer not null default 3, queue_cur_table integer not null default 0, @@ -78,6 +79,8 @@ create table pgq.queue ( queue_switch_time timestamptz not null default now(), queue_external_ticker boolean not null default false, + queue_disable_insert boolean not null default false, + queue_ticker_max_count integer not null default 500, queue_ticker_max_lag interval not null default '3 seconds', queue_ticker_idle_period interval not null default '1 minute', @@ -86,8 +89,8 @@ create table pgq.queue ( queue_event_seq text not null, queue_tick_seq text not null, - constraint queue_pkey primary key (queue_id), - constraint queue_name_uq unique (queue_name) + constraint queue_pkey primary key (queue_id), + constraint queue_name_uq unique (queue_name) ); -- ---------------------------------------------------------------------- @@ -100,14 +103,16 @@ create table pgq.queue ( -- tick_id - ticks id (per-queue) -- tick_time - time when tick happened -- tick_snapshot - transaction state +-- tick_event_seq - last value for event seq -- ---------------------------------------------------------------------- create table pgq.tick ( tick_queue int4 not null, tick_id bigint not null, tick_time timestamptz not null default now(), tick_snapshot txid_snapshot not null default txid_current_snapshot(), + tick_event_seq bigint not null, -- may be NULL on upgraded dbs - constraint tick_pkey primary key (tick_queue, tick_id), + constraint tick_pkey primary key (tick_queue, tick_id), constraint tick_queue_fkey foreign key (tick_queue) references pgq.queue (queue_id) ); @@ -134,16 +139,16 @@ create sequence pgq.batch_id_seq; -- sub_next_tick - batch end pos -- ---------------------------------------------------------------------- create table pgq.subscription ( - sub_id serial not null, - sub_queue int4 not null, - sub_consumer int4 not null, - sub_last_tick bigint not null, + sub_id serial not null, + sub_queue int4 not null, + sub_consumer int4 not null, + sub_last_tick bigint not null, sub_active timestamptz not null default now(), sub_batch bigint, sub_next_tick bigint, - constraint subscription_pkey primary key (sub_id), - constraint subscription_ukey unique (sub_queue, sub_consumer), + constraint subscription_pkey primary key (sub_id), + constraint subscription_ukey unique (sub_queue, sub_consumer), constraint sub_queue_fkey foreign key (sub_queue) references pgq.queue (queue_id), constraint sub_consumer_fkey foreign key (sub_consumer) @@ -170,7 +175,7 @@ create table pgq.subscription ( -- ev_extra4 - extra data field -- ---------------------------------------------------------------------- create table pgq.event_template ( - ev_id bigint not null, + ev_id bigint not null, ev_time timestamptz not null, ev_txid bigint not null default txid_current(), diff --git a/sql/pgq/structure/types.sql b/sql/pgq/structure/types.sql deleted file mode 100644 index 9b5c7438..00000000 --- a/sql/pgq/structure/types.sql +++ /dev/null @@ -1,39 +0,0 @@ - -create type pgq.ret_queue_info as ( - queue_name text, - queue_ntables integer, - queue_cur_table integer, - queue_rotation_period interval, - queue_switch_time timestamptz, - queue_external_ticker boolean, - queue_ticker_max_count integer, - queue_ticker_max_lag interval, - queue_ticker_idle_period interval, - ticker_lag interval -); - -create type pgq.ret_consumer_info as ( - queue_name text, - consumer_name text, - lag interval, - last_seen interval, - last_tick bigint, - current_batch bigint, - next_tick bigint -); - -create type pgq.ret_batch_event as ( - ev_id bigint, - ev_time timestamptz, - - ev_txid bigint, - ev_retry int4, - - ev_type text, - ev_data text, - ev_extra1 text, - ev_extra2 text, - ev_extra3 text, - ev_extra4 text -); - diff --git a/sql/pgq/structure/uninstall_pgq.sql b/sql/pgq/structure/uninstall_pgq.sql index 4ab28b30..61545b75 100644 --- a/sql/pgq/structure/uninstall_pgq.sql +++ b/sql/pgq/structure/uninstall_pgq.sql @@ -1,4 +1,4 @@ -- brute-force uninstall -drop schema cascade pgq; +drop schema pgq cascade; diff --git a/sql/pgq/triggers/Makefile b/sql/pgq/triggers/Makefile index 32ac6a05..9d5746a4 100644 --- a/sql/pgq/triggers/Makefile +++ b/sql/pgq/triggers/Makefile @@ -2,7 +2,7 @@ include ../../../config.mak MODULE_big = pgq_triggers -SRCS = logtriga.c logutriga.c sqltriga.c denytriga.c \ +SRCS = logtriga.c logutriga.c sqltriga.c \ common.c makesql.c stringutil.c OBJS = $(SRCS:.c=.o) DATA_built = pgq_triggers.sql diff --git a/sql/pgq/triggers/common.c b/sql/pgq/triggers/common.c index cd27ea19..7783d73c 100644 --- a/sql/pgq/triggers/common.c +++ b/sql/pgq/triggers/common.c @@ -198,11 +198,12 @@ full_reset(void) } /* - * Create new plan for insertion into current queue table. + * Fill table information in hash table. */ -static void -fill_tbl_info(Relation rel, struct PgqTableInfo *info) +static struct PgqTableInfo * +fill_tbl_info(Relation rel) { + struct PgqTableInfo *info; StringInfo pkeys; Datum values[1]; const char *name = pgq_find_table_name(rel); @@ -211,12 +212,24 @@ fill_tbl_info(Relation rel, struct PgqTableInfo *info) bool isnull; int res, i, attno; + /* load pkeys */ values[0] = ObjectIdGetDatum(rel->rd_id); res = SPI_execute_plan(pkey_plan, values, NULL, false, 0); if (res != SPI_OK_SELECT) elog(ERROR, "pkey_plan exec failed"); - desc = SPI_tuptable->tupdesc; + /* + * SPI_execute_plan may launch reset callback, thus we need + * to load the final position after calling it. + */ + init_module(); + info = hash_search(tbl_cache_map, &rel->rd_id, HASH_ENTER, NULL); + + /* + * Fill info + */ + + desc = SPI_tuptable->tupdesc; pkeys = makeStringInfo(); info->n_pkeys = SPI_processed; info->table_name = MemoryContextStrdup(tbl_cache_ctx, name); @@ -233,6 +246,8 @@ fill_tbl_info(Relation rel, struct PgqTableInfo *info) appendStringInfoString(pkeys, name); } info->pkey_list = MemoryContextStrdup(tbl_cache_ctx, pkeys->data); + + return info; } static void @@ -263,15 +278,15 @@ static void relcache_reset_cb(Datum arg, Oid relid) struct PgqTableInfo * pgq_find_table_info(Relation rel) { - struct PgqTableInfo *entry; - bool did_exist = false; + struct PgqTableInfo *entry; - init_module(); + init_module(); - entry = hash_search(tbl_cache_map, &rel->rd_id, HASH_ENTER, &did_exist); - if (!did_exist) - fill_tbl_info(rel, entry); - return entry; + entry = hash_search(tbl_cache_map, &rel->rd_id, HASH_FIND, NULL); + if (!entry) + entry = fill_tbl_info(rel); + + return entry; } static void @@ -439,3 +454,17 @@ bool pgqtriga_is_pkey(PgqTriggerEvent *ev, TriggerData *tg, int i, int attkind_i return false; } + +/* + * Check if trigger action should be skipped. + */ + +bool pgq_is_logging_disabled(void) +{ +#if defined(PG_VERSION_NUM) && PG_VERSION_NUM >= 80300 + if (SessionReplicationRole != SESSION_REPLICATION_ROLE_ORIGIN) + return true; +#endif + return false; +} + diff --git a/sql/pgq/triggers/common.h b/sql/pgq/triggers/common.h index 55319692..5be441b1 100644 --- a/sql/pgq/triggers/common.h +++ b/sql/pgq/triggers/common.h @@ -47,6 +47,8 @@ bool pgqtriga_skip_col(PgqTriggerEvent *ev, TriggerData *tg, int i, int attkind_ bool pgqtriga_is_pkey(PgqTriggerEvent *ev, TriggerData *tg, int i, int attkind_idx); void pgq_insert_tg_event(PgqTriggerEvent *ev); +bool pgq_is_logging_disabled(void); + /* makesql.c */ int pgqtriga_make_sql(PgqTriggerEvent *ev, TriggerData *tg, StringInfo sql); diff --git a/sql/pgq/triggers/denytriga.c b/sql/pgq/triggers/denytriga.c deleted file mode 100644 index 45927ef1..00000000 --- a/sql/pgq/triggers/denytriga.c +++ /dev/null @@ -1,75 +0,0 @@ -/* - * denytriga.c - Dumb deny trigger. - * - * Copyright (c) 2008 Marko Kreen, Skype Technologies OÜ - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -#include <postgres.h> - -#include <executor/spi.h> -#include <commands/trigger.h> -#include <utils/memutils.h> - -PG_FUNCTION_INFO_V1(pgq_denytriga); -Datum pgq_denytriga(PG_FUNCTION_ARGS); -PG_FUNCTION_INFO_V1(pgq_set_connection_context); -Datum pgq_set_connection_context(PG_FUNCTION_ARGS); - -static char *current_context = NULL; - -/* - * Connection context set. - */ - -Datum pgq_set_connection_context(PG_FUNCTION_ARGS) -{ - char *ctx; - if (current_context) - pfree(current_context); - current_context = NULL; - - if (PG_NARGS() > 0 && !PG_ARGISNULL(0)) { - ctx = DatumGetCString(DirectFunctionCall1(textout, PG_GETARG_DATUM(0))); - current_context = MemoryContextStrdup(TopMemoryContext, ctx); - pfree(ctx); - } - - PG_RETURN_VOID(); -} - -Datum -pgq_denytriga(PG_FUNCTION_ARGS) -{ - TriggerData *tg = (TriggerData *) (fcinfo->context); - - if (!CALLED_AS_TRIGGER(fcinfo)) - elog(ERROR, "pgq.denytriga not called as trigger"); - if (!TRIGGER_FIRED_AFTER(tg->tg_event)) - elog(ERROR, "pgq.denytriga must be fired AFTER"); - if (!TRIGGER_FIRED_FOR_ROW(tg->tg_event)) - elog(ERROR, "pgq.denytriga must be fired FOR EACH ROW"); - - if (current_context) { - int i; - for (i = 0; i < tg->tg_trigger->tgnargs; i++) { - char *arg = tg->tg_trigger->tgargs[i]; - if (strcmp(arg, current_context) == 0) - return PointerGetDatum(NULL); - } - } - - elog(ERROR, "action denied"); -} - diff --git a/sql/pgq/triggers/logtriga.c b/sql/pgq/triggers/logtriga.c index d27cfb54..b724dd60 100644 --- a/sql/pgq/triggers/logtriga.c +++ b/sql/pgq/triggers/logtriga.c @@ -56,6 +56,9 @@ pgq_logtriga(PG_FUNCTION_ARGS) if (!TRIGGER_FIRED_AFTER(tg->tg_event)) elog(ERROR, "pgq.logtriga must be fired AFTER"); + if (pgq_is_logging_disabled()) + goto skip_it; + /* * Connect to the SPI manager */ @@ -76,6 +79,7 @@ pgq_logtriga(PG_FUNCTION_ARGS) if (SPI_finish() < 0) elog(ERROR, "SPI_finish failed"); +skip_it: return PointerGetDatum(NULL); } diff --git a/sql/pgq/triggers/logutriga.c b/sql/pgq/triggers/logutriga.c index 17819c58..1dac75a5 100644 --- a/sql/pgq/triggers/logutriga.c +++ b/sql/pgq/triggers/logutriga.c @@ -95,6 +95,8 @@ pgq_logutriga(PG_FUNCTION_ARGS) else row = tg->tg_trigtuple; + if (pgq_is_logging_disabled()) + goto skip_it; /* * Connect to the SPI manager @@ -126,6 +128,7 @@ pgq_logutriga(PG_FUNCTION_ARGS) * After trigger ignores result, * before trigger skips event if NULL. */ +skip_it: if (TRIGGER_FIRED_AFTER(tg->tg_event) || ev.skip) return PointerGetDatum(NULL); else diff --git a/sql/pgq/triggers/pgq_triggers.sql.in b/sql/pgq/triggers/pgq_triggers.sql.in index 3edf1966..93f9ba7a 100644 --- a/sql/pgq/triggers/pgq_triggers.sql.in +++ b/sql/pgq/triggers/pgq_triggers.sql.in @@ -97,34 +97,3 @@ AS 'MODULE_PATHNAME', 'pgq_sqltriga' LANGUAGE C; CREATE OR REPLACE FUNCTION pgq.logutriga() RETURNS TRIGGER AS 'MODULE_PATHNAME', 'pgq_logutriga' LANGUAGE C; --- ---------------------------------------------------------------------- --- Function: pgq.denytriga() --- --- Trigger function that denies operation unless in particular context. --- --- Purpose: --- Used as producer for several PgQ standard consumers (cube_dispatcher, --- queue_mover, table_dispatcher). Basically for cases where the --- consumer wants to parse the event and look at the actual column values. --- --- Trigger parameters: --- One or more context names to allow. --- --- Regular listen trigger example: --- > CREATE TRIGGER triga_nimi AFTER INSERT OR UPDATE ON customer --- > FOR EACH ROW EXECUTE PROCEDURE pgq.denytriga('my_replica'); --- ---------------------------------------------------------------------- -CREATE OR REPLACE FUNCTION pgq.denytriga() RETURNS TRIGGER -AS 'MODULE_PATHNAME', 'pgq_denytriga' LANGUAGE C; - --- ---------------------------------------------------------------------- --- Function: pgq.set_connection_context(1) --- --- This function sets the context for current connections. --- --- Parameters: --- context - context name --- ---------------------------------------------------------------------- -CREATE OR REPLACE FUNCTION pgq.set_connection_context(i_context text) RETURNS void -AS 'MODULE_PATHNAME', 'pgq_set_connection_context' LANGUAGE C; - diff --git a/sql/pgq/triggers/sqltriga.c b/sql/pgq/triggers/sqltriga.c index 094cc319..6dbb1cd1 100644 --- a/sql/pgq/triggers/sqltriga.c +++ b/sql/pgq/triggers/sqltriga.c @@ -49,13 +49,17 @@ pgq_sqltriga(PG_FUNCTION_ARGS) if (!CALLED_AS_TRIGGER(fcinfo)) elog(ERROR, "pgq.logutriga not called as trigger"); + tg = (TriggerData *) (fcinfo->context); + + if (pgq_is_logging_disabled()) + goto skip_it; + /* * Connect to the SPI manager */ if (SPI_connect() < 0) elog(ERROR, "logtriga: SPI_connect() failed"); - tg = (TriggerData *) (fcinfo->context); pgq_prepare_event(&ev, tg, true); appendStringInfoChar(ev.ev_type, ev.op_type); @@ -74,6 +78,7 @@ pgq_sqltriga(PG_FUNCTION_ARGS) * After trigger ignores result, * before trigger skips event if NULL. */ +skip_it: if (TRIGGER_FIRED_AFTER(tg->tg_event) || ev.skip) return PointerGetDatum(NULL); else if (TRIGGER_FIRED_BY_UPDATE(tg->tg_event)) |