From 34c639525c1d7c25da67d13279131dffa6c950a2 Mon Sep 17 00:00:00 2001 From: Oleg Gurov Date: Fri, 1 Feb 2019 01:18:40 +0300 Subject: [PATCH 1/2] checkup on python prototype --- checkup-py/checks/__init__.py | 0 checkup-py/checks/indexes_invalid/__init__.py | 0 .../indexes_invalid/drop_create_ddls.sql | 12 ++ checkup-py/checks/indexes_invalid/impl.py | 25 +++ .../indexes_invalid/list_invalid_indexes.sql | 12 ++ checkup-py/common/__init__.py | 0 checkup-py/common/utils.py | 30 ++++ checkup-py/conf/postgres-checkup.conf | 0 checkup-py/postgres-checkup.py | 142 ++++++++++++++++++ checkup-py/readme.md | 45 ++++++ 10 files changed, 266 insertions(+) create mode 100644 checkup-py/checks/__init__.py create mode 100644 checkup-py/checks/indexes_invalid/__init__.py create mode 100644 checkup-py/checks/indexes_invalid/drop_create_ddls.sql create mode 100644 checkup-py/checks/indexes_invalid/impl.py create mode 100644 checkup-py/checks/indexes_invalid/list_invalid_indexes.sql create mode 100644 checkup-py/common/__init__.py create mode 100644 checkup-py/common/utils.py create mode 100644 checkup-py/conf/postgres-checkup.conf create mode 100644 checkup-py/postgres-checkup.py create mode 100644 checkup-py/readme.md diff --git a/checkup-py/checks/__init__.py b/checkup-py/checks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/checkup-py/checks/indexes_invalid/__init__.py b/checkup-py/checks/indexes_invalid/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/checkup-py/checks/indexes_invalid/drop_create_ddls.sql b/checkup-py/checks/indexes_invalid/drop_create_ddls.sql new file mode 100644 index 0000000..467647a --- /dev/null +++ b/checkup-py/checks/indexes_invalid/drop_create_ddls.sql @@ -0,0 +1,12 @@ +select + format('DROP INDEX CONCURRENTLY %s; -- %s, table %s', i.indexrelid::regclass::text, 'Invalid index', pct.relname) as drop_code, + replace( + format('%s; -- table %s', pg_get_indexdef(i.indexrelid), pct.relname), + 'CREATE INDEX', + 'CREATE INDEX CONCURRENTLY' + ) as revert_code +from pg_index i +join pg_class as pci on pci.oid = i.indexrelid +join pg_class as pct on pct.oid = i.indrelid +left join pg_namespace pn on pn.oid = pct.relnamespace +-- where i.indisvalid = false; -- disable to debug \ No newline at end of file diff --git a/checkup-py/checks/indexes_invalid/impl.py b/checkup-py/checks/indexes_invalid/impl.py new file mode 100644 index 0000000..fc76a98 --- /dev/null +++ b/checkup-py/checks/indexes_invalid/impl.py @@ -0,0 +1,25 @@ +import os +from common.utils import * + +class IndexesInvalid: + check_name = "indexes_invalid" + + def __init__(self): + self.current_dir = os.path.dirname(os.path.realpath(__file__)) + + def run(self, db_conn): + print("Hello from IndexesInvalid %s" % self.current_dir) + res = get_resultset( + db_conn, + get_file_content(os.path.join(self.current_dir, "list_invalid_indexes.sql")) + ) + print(str(res)) + + res = get_resultset( + db_conn, + get_file_content(os.path.join(self.current_dir, "drop_create_ddls.sql")) + ) + print(str(res)) + + def to_md(self): + pass diff --git a/checkup-py/checks/indexes_invalid/list_invalid_indexes.sql b/checkup-py/checks/indexes_invalid/list_invalid_indexes.sql new file mode 100644 index 0000000..7920752 --- /dev/null +++ b/checkup-py/checks/indexes_invalid/list_invalid_indexes.sql @@ -0,0 +1,12 @@ +select + coalesce(nullif(pn.nspname, 'public') || '.', '') || pct.relname as "relation_name", + pci.relname as index_name, + pn.nspname as schema_name, + pct.relname as table_name, + pg_size_pretty(pg_relation_size(i.indexrelid)) index_size +from pg_index i +join pg_class as pci on pci.oid = i.indexrelid +join pg_class as pct on pct.oid = i.indrelid +left join pg_namespace pn on pn.oid = pct.relnamespace +-- where i.indisvalid = false; -- disable to debug +order by pci.relname desc \ No newline at end of file diff --git a/checkup-py/common/__init__.py b/checkup-py/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/checkup-py/common/utils.py b/checkup-py/common/utils.py new file mode 100644 index 0000000..01c1366 --- /dev/null +++ b/checkup-py/common/utils.py @@ -0,0 +1,30 @@ +import traceback +import sys +import os + + +def exception_helper(): + exc_type, exc_value, exc_traceback = sys.exc_info() + return "\n".join( + [ + v for v in traceback.format_exception(exc_type, exc_value, exc_traceback) + ] + ) + + +def get_scalar(conn, query): + p_query = conn.prepare(query) + res = p_query() + return None if len(res) == 0 else next(row[0] for row in res) + + +def get_resultset(conn, query): + p_query = conn.prepare(query) + return p_query() + + +def get_file_content(file_name): + current_file = open(file_name, 'r') + file_content = current_file.read() + current_file.close() + return file_content \ No newline at end of file diff --git a/checkup-py/conf/postgres-checkup.conf b/checkup-py/conf/postgres-checkup.conf new file mode 100644 index 0000000..e69de29 diff --git a/checkup-py/postgres-checkup.py b/checkup-py/postgres-checkup.py new file mode 100644 index 0000000..0923353 --- /dev/null +++ b/checkup-py/postgres-checkup.py @@ -0,0 +1,142 @@ +from sshtunnel import SSHTunnelForwarder +from threading import Thread +import time +import argparse +import postgresql +from common.utils import * +from checks.indexes_invalid.impl import IndexesInvalid # each check must be imported here + + +class SysConf: + def __init__(self): + self.current_dir = os.path.dirname(os.path.realpath(__file__)) + self.application_name = 'postgres-checkup' + + +class CheckupGlobal: + sys_conf = None + args = None + common_report = [ # checks order defined + IndexesInvalid() + ] + check_names = {v.__class__.check_name:v for v in common_report} + + def __init__(self): + try: + parser = argparse.ArgumentParser() + parser.add_argument( + "--db-name", + type=str, + default="test" + ) + parser.add_argument( + "--db-remote-port", + type=int, + default=5432 + ) + parser.add_argument( + "--db-local-port", + type=int, + default=5400 + ) + parser.add_argument( + "--db-user-name", + type=str, + default="postgres" + ) + parser.add_argument( + "--db-user-password", + type=str, + default="postgres" + ) + + parser.add_argument("--ssh-host", type=str) + parser.add_argument("--ssh-port", type=str, default="22") + parser.add_argument("--ssh-user", type=str) + parser.add_argument("--ssh-password", type=str) + + parser.add_argument("--check", type=str) + + try: + self.args = parser.parse_args() + except: + print(exception_helper()) + sys.exit(0) + + if not len(sys.argv) > 1: + print("No arguments. Type -h for help.") + sys.exit(0) + + self.sys_conf = SysConf() + except SystemExit as e: + print("Exiting...") + sys.exit(0) + except: + print(exception_helper()) + print("Can't initialize application. Exiting...") + sys.exit(0) + + +def run_checks(): + str_conn = 'pq://%s:%s@%s:%s/%s' % ( + Checkup.args.db_user_name, + Checkup.args.db_user_password, + '127.0.0.1', + Checkup.args.db_local_port, + Checkup.args.db_name + ) + db_conn = postgresql.open(str_conn) + + get_resultset(db_conn, "select 1, 2, 3") + if Checkup.args.check == "ALL": + # TODO + pass + else: + Checkup.check_names[Checkup.args.check].run(db_conn) + + db_conn.close() + + +if __name__ == "__main__": + print('===========> checkup started') + Checkup = CheckupGlobal() # global object with configuration and logger + + try: + server = SSHTunnelForwarder( + Checkup.args.ssh_host, + ssh_port=Checkup.args.ssh_port, + ssh_username=Checkup.args.ssh_user, + ssh_password=Checkup.args.ssh_password, + remote_bind_address=('127.0.0.1', Checkup.args.db_remote_port), + local_bind_address=('127.0.0.1', Checkup.args.db_local_port) + ) + + server.start() + except: + print(exception_helper()) + print("Can't start SSHTunnelForwarder. Exiting...") + sys.exit(0) + + str_conn = 'pq://%s:%s@%s:%s/%s' % ( + Checkup.args.db_user_name, + Checkup.args.db_user_password, + '127.0.0.1', + Checkup.args.db_local_port, + Checkup.args.db_name + ) + + worker_threads = [] + print('Start threads initialization for host "%s"' % Checkup.args.ssh_host) + worker_threads.append(Thread(target=run_checks, args=[])) + + for thread in worker_threads: thread.start() + alive_count = 1 + print('Threads successfully initialized for host %s' % Checkup.args.ssh_host) + while alive_count > 0: + alive_count = len([thread for thread in worker_threads if thread.is_alive()]) + if alive_count == 0: break + print('Live %s threads' % alive_count) + time.sleep(1) + + server.stop() + print('<========== checkup finished') \ No newline at end of file diff --git a/checkup-py/readme.md b/checkup-py/readme.md new file mode 100644 index 0000000..0521ae5 --- /dev/null +++ b/checkup-py/readme.md @@ -0,0 +1,45 @@ +# Prepare + +On windows + +```bash +# install python +https://www.python.org/ftp/python/3.6.8/python-3.6.8-amd64.exe +python -m pip install --upgrade pip + +cd C:\Program Files\Python36\Scripts +# or +cd C:\Users\USER\AppData\Local\Programs\Python\Python36\Scripts\ + +pip3.6.exe install py-postgresql +pip3.6.exe install paramiko +pip3.6.exe install sshtunnel +``` + +On RHEL + +```bash +# install python +yum install -y python36 +# if pip not installed +curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py +python3.6 get-pip.py +# else update pip +python3.6 -m pip install --upgrade pip + +pip3.6 install py-postgresql +pip3.6 install paramiko +pip3.6 install sshtunnel +``` + +# Usage + +```bash +python3.6 postgres-checkup.py \ + --ssh-host=194.67.206.177 \ + --ssh-port=22 \ + --ssh-user=root \ + --ssh-password=*** \ + --db-name=test \ + --check=indexes_invalid +``` -- GitLab From 184efe62f464bb0979b55f9ac1eb1cda7734d383 Mon Sep 17 00:00:00 2001 From: Oleg Gurov Date: Sat, 2 Feb 2019 00:33:52 +0300 Subject: [PATCH 2/2] output to md --- checkup-py/checks/indexes_invalid/impl.py | 54 +++++++++++++------ .../indexes_invalid/list_invalid_indexes.sql | 5 +- checkup-py/common/utils.py | 7 ++- checkup-py/postgres-checkup.py | 10 ++-- 4 files changed, 52 insertions(+), 24 deletions(-) diff --git a/checkup-py/checks/indexes_invalid/impl.py b/checkup-py/checks/indexes_invalid/impl.py index fc76a98..7161896 100644 --- a/checkup-py/checks/indexes_invalid/impl.py +++ b/checkup-py/checks/indexes_invalid/impl.py @@ -4,22 +4,44 @@ from common.utils import * class IndexesInvalid: check_name = "indexes_invalid" - def __init__(self): + def __init__(self, ctx): self.current_dir = os.path.dirname(os.path.realpath(__file__)) + self.ctx = ctx def run(self, db_conn): - print("Hello from IndexesInvalid %s" % self.current_dir) - res = get_resultset( - db_conn, - get_file_content(os.path.join(self.current_dir, "list_invalid_indexes.sql")) - ) - print(str(res)) - - res = get_resultset( - db_conn, - get_file_content(os.path.join(self.current_dir, "drop_create_ddls.sql")) - ) - print(str(res)) - - def to_md(self): - pass + # print("Hello from IndexesInvalid %s" % self.current_dir) + tbl_1 = get_resultset(db_conn, get_file_content(os.path.join(self.current_dir, "list_invalid_indexes.sql"))) + tbl_2 = get_resultset(db_conn, get_file_content(os.path.join(self.current_dir, "drop_create_ddls.sql"))) + self.to_md(tbl_1, tbl_2) + + def to_md(self, tbl_1, tbl_2): + md_content = "" + md_content += "# Indexes -> invalid #\n\n" + md_content += "## Observations ##\n\n" + md_content += "### Master %s ###\n" % self.ctx.args.db_name + + if len(tbl_1) > 0: + md_content += "Schema name | Table name | Index name | Index size\n" + md_content += "------------|------------|------------|------------\n" + for row in tbl_1: + md_content += "%s | %s | %s | %s\n" % (row[0], row[1], row[2], row[3]) + else: + md_content += "Invalid indexes not found\n" + + md_content += "\n\n## Recommendations ##\n\n" + + if len(tbl_2) > 0: + md_content += """\n\n#### "DO" database migration code ####\n\n""" + md_content += "```\n" + md_content += """-- Call each line separately. "CONCURRENTLY" queries cannot be\n""" + md_content += """-- combined in multi-statement requests.\n""" + for row in tbl_2: + md_content += "%s\n" % (row[0]) + md_content += "```\n" + md_content += """\n\n#### "UNDO" database migration code ####\n\n""" + md_content += "```\n" + for row in tbl_2: + md_content += "%s\n" % (row[1]) + md_content += "```\n" + + save_to(os.path.join(self.ctx.sys_conf.current_dir, "output", self.check_name + ".md"), md_content) \ No newline at end of file diff --git a/checkup-py/checks/indexes_invalid/list_invalid_indexes.sql b/checkup-py/checks/indexes_invalid/list_invalid_indexes.sql index 7920752..aac9820 100644 --- a/checkup-py/checks/indexes_invalid/list_invalid_indexes.sql +++ b/checkup-py/checks/indexes_invalid/list_invalid_indexes.sql @@ -1,8 +1,7 @@ select - coalesce(nullif(pn.nspname, 'public') || '.', '') || pct.relname as "relation_name", - pci.relname as index_name, pn.nspname as schema_name, - pct.relname as table_name, + pct.relname as table_name, + pci.relname as index_name, pg_size_pretty(pg_relation_size(i.indexrelid)) index_size from pg_index i join pg_class as pci on pci.oid = i.indexrelid diff --git a/checkup-py/common/utils.py b/checkup-py/common/utils.py index 01c1366..6444696 100644 --- a/checkup-py/common/utils.py +++ b/checkup-py/common/utils.py @@ -27,4 +27,9 @@ def get_file_content(file_name): current_file = open(file_name, 'r') file_content = current_file.read() current_file.close() - return file_content \ No newline at end of file + return file_content + +def save_to(filename, contents): + fh = open(filename, 'w') + fh.write(contents) + fh.close() diff --git a/checkup-py/postgres-checkup.py b/checkup-py/postgres-checkup.py index 0923353..703a793 100644 --- a/checkup-py/postgres-checkup.py +++ b/checkup-py/postgres-checkup.py @@ -16,10 +16,8 @@ class SysConf: class CheckupGlobal: sys_conf = None args = None - common_report = [ # checks order defined - IndexesInvalid() - ] - check_names = {v.__class__.check_name:v for v in common_report} + common_report = None + check_names = None def __init__(self): try: @@ -68,6 +66,10 @@ class CheckupGlobal: sys.exit(0) self.sys_conf = SysConf() + self.common_report = [ # checks order defined + IndexesInvalid(self) + ] + self.check_names = {v.__class__.check_name:v for v in self.common_report} except SystemExit as e: print("Exiting...") sys.exit(0) -- GitLab