diff options
author | Marko Kreen | 2011-08-30 11:26:48 +0000 |
---|---|---|
committer | Marko Kreen | 2011-08-30 11:26:48 +0000 |
commit | 24c02c812349d76541703149ca8427bf34a32b4c (patch) | |
tree | 75390dcdf5e18c40f2cccb09512f4d7553bf628a | |
parent | 83fd021a904c8c967fb0ac42766783fbf2ae6c6c (diff) |
londiste.handlers.bulk: hack - try to use real table
-rw-r--r-- | python/londiste/handlers/bulk.py | 34 |
1 files changed, 25 insertions, 9 deletions
diff --git a/python/londiste/handlers/bulk.py b/python/londiste/handlers/bulk.py index 5f1177d5..a9d0d3f6 100644 --- a/python/londiste/handlers/bulk.py +++ b/python/londiste/handlers/bulk.py @@ -41,6 +41,8 @@ DEFAULT_METHOD = METH_CORRECT AVOID_BIZGRES_BUG = 0 USE_LONGLIVED_TEMP_TABLES = True +USE_REAL_TABLE = True + class BulkEvent(object): """Helper class for BulkLoader to store relevant data.""" __slots__ = ('op', 'data', 'pk_data') @@ -191,11 +193,9 @@ class BulkLoader(BaseHandler): ",".join(self.pkey_list), ",".join(self.dist_fields))) # create temp table - temp = self.create_temp_table(curs) + temp, qtemp = self.create_temp_table(curs) tbl = self.table_name - qtbl = quote_fqident(self.table_name) - qtemp = quote_ident(temp) # where expr must have pkey and dist fields klist = [] @@ -291,7 +291,7 @@ class BulkLoader(BaseHandler): # delete remaining rows if temp_used: - if USE_LONGLIVED_TEMP_TABLES: + if USE_LONGLIVED_TEMP_TABLES or USE_REAL_TABLE: q = "truncate %s" % qtemp else: # fscking problems with long-lived temp tables @@ -302,14 +302,30 @@ class BulkLoader(BaseHandler): self.reset() def create_temp_table(self, curs): - # create temp table for loading - tempname = self.table_name.replace('.', '_') + "_loadertmp" + if USE_REAL_TABLE: + tempname = self.table_name + "_loadertmpx" + else: + # create temp table for loading + tempname = self.table_name.replace('.', '_') + "_loadertmp" # check if exists - if USE_LONGLIVED_TEMP_TABLES: + if USE_REAL_TABLE: + if skytools.exists_table(curs, tempname): + self.log.debug("bulk: Using existing real table %s" % tempname) + return tempname, quote_fqident(tempname) + + # create non-temp table + q = "create table %s (like %s) %s" % ( + quote_fqident(tempname), + quote_fqident(self.table_name), + arg) + self.log.debug("bulk: Creating real table: %s" % q) + curs.execute(q) + return tempname, quote_fqident(tempname) + elif USE_LONGLIVED_TEMP_TABLES: if skytools.exists_temp_table(curs, tempname): self.log.debug("bulk: Using existing temp table %s" % tempname) - return tempname + return tempname, quote_ident(tempname) # bizgres crashes on delete rows # removed arg = "on commit delete rows" @@ -319,7 +335,7 @@ class BulkLoader(BaseHandler): quote_ident(tempname), quote_fqident(self.table_name), arg) self.log.debug("bulk: Creating temp table: %s" % q) curs.execute(q) - return tempname + return tempname, quote_ident(tempname) def find_dist_fields(self, curs): if not skytools.exists_table(curs, "pg_catalog.gp_distribution_policy"): |