sql upgrade cleanup

- use 'structure/upgrade.sql' to generate X.upgrade.sql
- add pgq_ext to upgraded schemas
- move non-standard upgrade scripts to extra/
- setup_skytools: build sql files in 'build' target, not always
- pgq_ext.version: bump ver to 3.0.0.1

15 files changed, 50 insertions, 25 deletions

diff --git a/misc/bumpver.sh b/misc/bumpver.sh
index 5bd901ed..ae9bf7db 100755
--- a/misc/bumpver.sh
+++ b/misc/bumpver.sh
@@ -1,6 +1,6 @@
 #! /bin/sh
 
-for s in pgq pgq_node pgq_coop londiste; do
+for s in pgq pgq_node pgq_coop londiste pgq_ext; do
   sver=$(git log --raw -n 1 sql/$s/functions | head -1)
   fn="sql/$s/functions/$s.version.sql"
   fver=$(git log --raw -n 1 "$fn" | head -1)
diff --git a/scripts/skytools_upgrade.py b/scripts/skytools_upgrade.py
index 37aa779c..169a1913 100755
--- a/scripts/skytools_upgrade.py
+++ b/scripts/skytools_upgrade.py
@@ -16,7 +16,7 @@ from skytools.natsort import natsort_key
 
 
 # schemas, where .upgrade.sql is enough
-AUTO_UPGRADE = ('pgq', 'pgq_node', 'pgq_coop', 'londiste')
+AUTO_UPGRADE = ('pgq', 'pgq_node', 'pgq_coop', 'londiste', 'pgq_ext')
 
 # fetch list of databases
 DB_LIST = "select datname from pg_database "\
@@ -28,6 +28,7 @@ version_list = [
     # schema, ver, filename, recheck_func
     ['pgq', '3.0', None, None],
     ['londiste', '3.0', None, None],
+    ['pgq_ext', '2.1', None, None],
 ]
 
 
diff --git a/setup_skytools.py b/setup_skytools.py
index 14ce9a6e..e5dd97e5 100755
--- a/setup_skytools.py
+++ b/setup_skytools.py
@@ -60,13 +60,17 @@ sql_files = [
    'sql/pgq_node/pgq_node.upgrade.sql',
    'sql/londiste/londiste.upgrade.sql',
    'sql/pgq_coop/pgq_coop.upgrade.sql',
+   'sql/pgq_ext/pgq_ext.upgrade.sql',
+]
 
+# sql files for special occasions
+extra_sql_files = [
    'upgrade/final/v3.0_pgq_core.sql',
-   #'sql/txid/txid.sql',
 ]
 
 if not INSTALL_SQL:
     sql_files = []
+    extra_sql_files = []
 
 def getvar(name, default):
     try:
@@ -92,15 +96,21 @@ DEF_SKYLOG = getvar('SKYLOG', '0') != '0'
 DEF_SK3_SUBDIR = getvar('SK3_SUBDIR', '0') != '0'
 
 # create sql files if they dont exist
-for fn in sql_files:
-    if not os.path.isfile(fn):
-        f = open(fn, 'w')
-        wd = os.path.dirname(fn)
-        cmd = [sys.executable, '../../scripts/catsql.py', 'structure/install.sql']
-        p = Popen(cmd, stdout=f, cwd = wd)
-        p.communicate()
-        if p.returncode != 0:
-            raise Exception('catsql failed')
+def make_sql():
+    for fn in sql_files:
+        if not os.path.isfile(fn):
+            f = open(fn, 'w')
+            wd = os.path.dirname(fn)
+            if fn.endswith('upgrade.sql'):
+                base = 'structure/upgrade.sql'
+            else:
+                base = 'structure/install.sql'
+            print("Creating %s" % (fn,))
+            cmd = [sys.executable, '../../scripts/catsql.py', base]
+            p = Popen(cmd, stdout=f, cwd = wd)
+            p.communicate()
+            if p.returncode != 0:
+                raise Exception('catsql failed')
 
 # remove .py, add suffix
 def fixscript(fn, dstdir, sfx):
@@ -121,9 +131,14 @@ class sk3_build(build):
         build.initialize_options(self)
         self.build_base = 'build.sk3'
 
+    def run(self):
+        build.run(self)
+        make_sql()
+
 # fix script names in build dir
 class sk3_build_scripts(build_scripts):
     def run(self):
+
         build_scripts.run(self)
 
         for sfn in sfx_scripts:
@@ -142,7 +157,6 @@ class sk3_install(install):
     skylog = DEF_SKYLOG
 
     def run(self):
-
         # create installer_config.py with final paths
         fn = 'python/skytools/installer_config.py'
         cf = open(fn + '.in', 'r').read()
@@ -182,7 +196,8 @@ setup(
         'python/conf/wal-master.ini',
         'python/conf/wal-slave.ini',
         ]),
-      ('share/skytools3', sql_files)],
+      ('share/skytools3', sql_files),
+      ('share/skytools3/extra', extra_sql_files)],
     ext_modules = c_modules,
     scripts = sfx_scripts + nosfx_scripts,
     cmdclass = {
diff --git a/sql/londiste/Makefile b/sql/londiste/Makefile
index 3fcae21a..c81efb34 100644
--- a/sql/londiste/Makefile
+++ b/sql/londiste/Makefile
@@ -25,7 +25,7 @@ londiste.sql: $(SRCS)
 	$(CATSQL) $(SQLS) > $@
 
 londiste.upgrade.sql: $(SRCS)
-	$(CATSQL) structure/functions.sql > $@
+	$(CATSQL) structure/upgrade.sql > $@
 
 test: londiste.sql
 	$(MAKE) installcheck || { filterdiff --format=unified regression.diffs | less; exit 1; }
diff --git a/sql/londiste/structure/upgrade.sql b/sql/londiste/structure/upgrade.sql
new file mode 100644
index 00000000..80b65496
--- /dev/null
+++ b/sql/londiste/structure/upgrade.sql
@@ -0,0 +1 @@
+\i structure/functions.sql
diff --git a/sql/pgq/Makefile b/sql/pgq/Makefile
index 19759831..cfee2a77 100644
--- a/sql/pgq/Makefile
+++ b/sql/pgq/Makefile
@@ -44,7 +44,7 @@ pgq.sql: $(SRCS)
 
 # fixme: use LDRS here, to upgrade triggers
 pgq.upgrade.sql: $(SRCS)
-	$(CATSQL) structure/func_internal.sql structure/func_public.sql > $@
+	$(CATSQL) structure/upgrade.sql > $@
 
 #
 # docs
diff --git a/sql/pgq/structure/upgrade.sql b/sql/pgq/structure/upgrade.sql
new file mode 100644
index 00000000..1b898d11
--- /dev/null
+++ b/sql/pgq/structure/upgrade.sql
@@ -0,0 +1,2 @@
+\i structure/func_internal.sql
+\i structure/func_public.sql
diff --git a/sql/pgq_coop/Makefile b/sql/pgq_coop/Makefile
index 0affc96b..31c5d329 100644
--- a/sql/pgq_coop/Makefile
+++ b/sql/pgq_coop/Makefile
@@ -2,7 +2,6 @@
 DATA_built = pgq_coop.sql pgq_coop.upgrade.sql
 
 SQL_FULL = structure/schema.sql structure/functions.sql structure/grants.sql
-SQL_UPGRADE = structure/functions.sql
 
 FUNCS = $(shell sed -n -e '/^\\/{s/\\i //;p}' structure/functions.sql)
 SRCS = $(SQL_FULL) $(FUNCS)
@@ -26,7 +25,7 @@ pgq_coop.sql: $(SRCS)
 	$(CATSQL) structure/install.sql > $@
 
 pgq_coop.upgrade.sql: $(SRCS)
-	$(CATSQL) $(SQL_UPGRADE) > $@
+	$(CATSQL) structure/upgrade.sql > $@
 
 #
 # docs
diff --git a/sql/pgq_coop/structure/upgrade.sql b/sql/pgq_coop/structure/upgrade.sql
new file mode 100644
index 00000000..80b65496
--- /dev/null
+++ b/sql/pgq_coop/structure/upgrade.sql
@@ -0,0 +1 @@
+\i structure/functions.sql
diff --git a/sql/pgq_ext/Makefile b/sql/pgq_ext/Makefile
index b76db050..3ab3cd4c 100644
--- a/sql/pgq_ext/Makefile
+++ b/sql/pgq_ext/Makefile
@@ -1,6 +1,6 @@
 
 DOCS = README.pgq_ext
-DATA_built = pgq_ext.sql
+DATA_built = pgq_ext.sql pgq_ext.upgrade.sql
 
 SRCS = structure/tables.sql functions/track_batch.sql functions/track_event.sql \
        functions/track_tick.sql functions/version.sql
@@ -17,6 +17,9 @@ include $(PGXS)
 pgq_ext.sql: $(SRCS)
 	$(CATSQL) structure/install.sql > $@
 
+pgq_ext.upgrade.sql: $(SRCS)
+	$(CATSQL) structure/upgrade.sql > $@
+
 test: pgq_ext.sql
 	make installcheck || { less regression.diffs ; exit 1; }
 
diff --git a/sql/pgq_ext/functions/version.sql b/sql/pgq_ext/functions/version.sql
index 5e6be8ac..de9081f5 100644
--- a/sql/pgq_ext/functions/version.sql
+++ b/sql/pgq_ext/functions/version.sql
@@ -2,7 +2,7 @@
 create or replace function pgq_ext.version()
 returns text as $$
 begin
-    return '2.1.6';
+    return '3.0.0.1';
 end;
 $$ language plpgsql;
 
diff --git a/sql/pgq_ext/structure/install.sql b/sql/pgq_ext/structure/install.sql
index 30c87741..973e35b8 100644
--- a/sql/pgq_ext/structure/install.sql
+++ b/sql/pgq_ext/structure/install.sql
@@ -1,6 +1,3 @@
 \i structure/tables.sql
-\i functions/track_batch.sql
-\i functions/track_event.sql
-\i functions/track_tick.sql
-\i functions/version.sql
+\i structure/upgrade.sql
 
diff --git a/sql/pgq_ext/structure/upgrade.sql b/sql/pgq_ext/structure/upgrade.sql
new file mode 100644
index 00000000..baafbce7
--- /dev/null
+++ b/sql/pgq_ext/structure/upgrade.sql
@@ -0,0 +1,5 @@
+\i functions/track_batch.sql
+\i functions/track_event.sql
+\i functions/track_tick.sql
+\i functions/version.sql
+
diff --git a/sql/pgq_node/Makefile b/sql/pgq_node/Makefile
index 1d88ce6a..2d24fcb3 100644
--- a/sql/pgq_node/Makefile
+++ b/sql/pgq_node/Makefile
@@ -24,7 +24,7 @@ pgq_node.sql: $(SRCS)
 	$(CATSQL) structure/install.sql > $@
 
 pgq_node.upgrade.sql: $(SRCS)
-	$(CATSQL) structure/functions.sql > $@
+	$(CATSQL) structure/upgrade.sql > $@
 
 #
 # docs
diff --git a/sql/pgq_node/structure/upgrade.sql b/sql/pgq_node/structure/upgrade.sql
new file mode 100644
index 00000000..80b65496
--- /dev/null
+++ b/sql/pgq_node/structure/upgrade.sql
@@ -0,0 +1 @@
+\i structure/functions.sql