Skip to content

Core: Add view-override catalog property #12534

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nastra merged 3 commits into from
Mar 18, 2025
Merged

Core: Add view-override catalog property #12534

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d080ad0
Core: Add view-override catalog property
ebyhr Mar 15, 2025
09592c2
fixup! Core: Add view-override catalog property
ebyhr Mar 17, 2025
930569e
fixup! Core: Add view-override catalog property
ebyhr Mar 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions core/src/main/java/org/apache/iceberg/CatalogProperties.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ private CatalogProperties() {}
public static final String TABLE_DEFAULT_PREFIX = "table-default.";
public static final String TABLE_OVERRIDE_PREFIX = "table-override.";
public static final String VIEW_DEFAULT_PREFIX = "view-default.";
public static final String VIEW_OVERRIDE_PREFIX = "view-override.";
public static final String METRICS_REPORTER_IMPL = "metrics-reporter-impl";

/**
Expand Down
18 changes: 18 additions & 0 deletions core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1205,6 +1205,20 @@ private Map<String, String> viewDefaultProperties() {
return viewDefaultProperties;
}

/**
* Get view properties that are enforced at Catalog level through catalog properties.
*
* @return overriding view properties enforced through catalog properties
*/
private Map<String, String> viewOverrideProperties() {
Map<String, String> viewOverrideProperties =
PropertyUtil.propertiesWithPrefix(properties(), CatalogProperties.VIEW_OVERRIDE_PREFIX);
LOG.info(
"View properties enforced at catalog level through catalog properties: {}",
viewOverrideProperties);
return viewOverrideProperties;
}

@Override
public ViewBuilder withSchema(Schema newSchema) {
this.schema = newSchema;
Expand Down Expand Up @@ -1268,6 +1282,8 @@ public View create() {
.putAllSummary(EnvironmentContext.get())
.build();

properties.putAll(viewOverrideProperties());

CreateViewRequest request =
ImmutableCreateViewRequest.builder()
.name(identifier.name())
Expand Down Expand Up @@ -1365,6 +1381,8 @@ private View replace(LoadViewResponse response) {
.putAllSummary(EnvironmentContext.get())
.build();

properties.putAll(viewOverrideProperties());

ViewMetadata.Builder builder =
ViewMetadata.buildFrom(metadata)
.setProperties(properties)
Expand Down
18 changes: 18 additions & 0 deletions core/src/main/java/org/apache/iceberg/view/BaseMetastoreViewCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,20 @@ private Map<String, String> viewDefaultProperties() {
return viewDefaultProperties;
}

/**
* Get view properties that are enforced at Catalog level through catalog properties.
*
* @return overriding view properties enforced through catalog properties
*/
private Map<String, String> viewOverrideProperties() {
Map<String, String> viewOverrideProperties =
PropertyUtil.propertiesWithPrefix(properties(), CatalogProperties.VIEW_OVERRIDE_PREFIX);
LOG.info(
"View properties enforced at catalog level through catalog properties: {}",
viewOverrideProperties);
return viewOverrideProperties;
}

@Override
public ViewBuilder withSchema(Schema newSchema) {
this.schema = newSchema;
Expand Down Expand Up @@ -187,6 +201,8 @@ private View create(ViewOperations ops) {
.putAllSummary(EnvironmentContext.get())
.build();

properties.putAll(viewOverrideProperties());

ViewMetadata viewMetadata =
ViewMetadata.builder()
.setProperties(properties)
Expand Down Expand Up @@ -236,6 +252,8 @@ private View replace(ViewOperations ops) {
.putAllSummary(EnvironmentContext.get())
.build();

properties.putAll(viewOverrideProperties());

ViewMetadata.Builder builder =
ViewMetadata.buildFrom(metadata)
.setProperties(properties)
Expand Down
3 changes: 3 additions & 0 deletions core/src/test/java/org/apache/iceberg/inmemory/TestInMemoryViewCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ public void before() {
ImmutableMap.<String, String>builder()
.put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1")
.put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", "catalog-default-key2")
.put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key3", "catalog-default-key3")
.put(CatalogProperties.VIEW_OVERRIDE_PREFIX + "key3", "catalog-override-key3")
.put(CatalogProperties.VIEW_OVERRIDE_PREFIX + "key4", "catalog-override-key4")
.build());
}

Expand Down
3 changes: 3 additions & 0 deletions core/src/test/java/org/apache/iceberg/jdbc/TestJdbcViewCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ public void before() {
properties.put(JdbcUtil.SCHEMA_VERSION_PROPERTY, JdbcUtil.SchemaVersion.V1.name());
properties.put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1");
properties.put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", "catalog-default-key2");
properties.put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key3", "catalog-default-key3");
properties.put(CatalogProperties.VIEW_OVERRIDE_PREFIX + "key3", "catalog-override-key3");
properties.put(CatalogProperties.VIEW_OVERRIDE_PREFIX + "key4", "catalog-override-key4");

catalog = new JdbcCatalog();
catalog.setConf(new Configuration());
Expand Down
3 changes: 3 additions & 0 deletions core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,9 @@ public void createCatalog() throws Exception {
.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse.getAbsolutePath())
.put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1")
.put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", "catalog-default-key2")
.put(CatalogProperties.VIEW_DEFAULT_PREFIX + "key3", "catalog-default-key3")
.put(CatalogProperties.VIEW_OVERRIDE_PREFIX + "key3", "catalog-override-key3")
.put(CatalogProperties.VIEW_OVERRIDE_PREFIX + "key4", "catalog-override-key4")
.build());

RESTCatalogAdapter adaptor =
Expand Down
8 changes: 7 additions & 1 deletion core/src/test/java/org/apache/iceberg/rest/TestRESTViewCatalogWithAssumedViewSupport.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,12 @@ public <T extends RESTResponse> T handleRequest(
CatalogProperties.VIEW_DEFAULT_PREFIX + "key1",
"catalog-default-key1",
CatalogProperties.VIEW_DEFAULT_PREFIX + "key2",
"catalog-default-key2"));
"catalog-default-key2",
CatalogProperties.VIEW_DEFAULT_PREFIX + "key3",
"catalog-default-key3",
CatalogProperties.VIEW_OVERRIDE_PREFIX + "key3",
"catalog-override-key3",
CatalogProperties.VIEW_OVERRIDE_PREFIX + "key4",
"catalog-override-key4"));
}
}
33 changes: 33 additions & 0 deletions core/src/test/java/org/apache/iceberg/view/ViewCatalogTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,39 @@ public void defaultViewProperties() {
assertThat(catalog().viewExists(identifier)).as("View should not exist").isFalse();
}

@Test
public void overrideViewProperties() {
TableIdentifier identifier = TableIdentifier.of("ns", "view");

if (requiresNamespaceCreate()) {
catalog().createNamespace(identifier.namespace());
}

assertThat(catalog().viewExists(identifier)).as("View should not exist").isFalse();

View view =
catalog()
.buildView(identifier)
.withSchema(SCHEMA)
.withDefaultNamespace(identifier.namespace())
.withDefaultCatalog(catalog().name())
.withQuery("spark", "select * from ns.tbl")
.withProperty("key4", "catalog-overridden-key4")
.withProperty("prop1", "val1")
.create();

assertThat(view).isNotNull();
assertThat(view.properties())
.containsEntry("key1", "catalog-default-key1")
.containsEntry("key2", "catalog-default-key2")
.containsEntry("key3", "catalog-override-key3")
.containsEntry("key4", "catalog-override-key4")
.containsEntry("prop1", "val1");

assertThat(catalog().dropView(identifier)).isTrue();
assertThat(catalog().viewExists(identifier)).as("View should not exist").isFalse();
}

@Test
public void completeCreateView() {
TableIdentifier identifier = TableIdentifier.of("ns", "view");
Expand Down
1 change: 1 addition & 0 deletions docs/docs/spark-configuration.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ Both catalogs are configured using properties nested under the catalog name. Com
| spark.sql.catalog._catalog-name_.table-default._propertyKey_ | | Default Iceberg table property value for property key _propertyKey_, which will be set on tables created by this catalog if not overridden |
| spark.sql.catalog._catalog-name_.table-override._propertyKey_ | | Enforced Iceberg table property value for property key _propertyKey_, which cannot be overridden on table creation by user |
| spark.sql.catalog._catalog-name_.view-default._propertyKey_ | | Default Iceberg view property value for property key _propertyKey_, which will be set on views created by this catalog if not overridden |
| spark.sql.catalog._catalog-name_.view-override._propertyKey_ | | Enforced Iceberg view property value for property key _propertyKey_, which cannot be overridden on view creation by user |
| spark.sql.catalog._catalog-name_.use-nullable-query-schema | `true` or `false` | Whether to preserve fields' nullability when creating the table using CTAS and RTAS. If set to `true`, all fields will be marked as nullable. If set to `false`, fields' nullability will be preserved. The default value is `true`. Available in Spark 3.5 and above. |

Additional properties can be found in common [catalog configuration](configuration.md#catalog-properties).
Expand Down
8 changes: 7 additions & 1 deletion hive-metastore/src/test/java/org/apache/iceberg/hive/TestHiveViewCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,13 @@ public void before() throws TException {
CatalogProperties.VIEW_DEFAULT_PREFIX + "key1",
"catalog-default-key1",
CatalogProperties.VIEW_DEFAULT_PREFIX + "key2",
"catalog-default-key2"),
"catalog-default-key2",
CatalogProperties.VIEW_DEFAULT_PREFIX + "key3",
"catalog-default-key3",
CatalogProperties.VIEW_OVERRIDE_PREFIX + "key3",
"catalog-override-key3",
CatalogProperties.VIEW_OVERRIDE_PREFIX + "key4",
"catalog-override-key4"),
HIVE_METASTORE_EXTENSION.hiveConf());
}

Expand Down
8 changes: 7 additions & 1 deletion nessie/src/test/java/org/apache/iceberg/nessie/TestNessieViewCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,13 @@ private NessieCatalog initNessieCatalog(String ref) {
CatalogProperties.VIEW_DEFAULT_PREFIX + "key1",
"catalog-default-key1",
CatalogProperties.VIEW_DEFAULT_PREFIX + "key2",
"catalog-default-key2");
"catalog-default-key2",
CatalogProperties.VIEW_DEFAULT_PREFIX + "key3",
"catalog-default-key3",
CatalogProperties.VIEW_OVERRIDE_PREFIX + "key3",
"catalog-override-key3",
CatalogProperties.VIEW_OVERRIDE_PREFIX + "key4",
"catalog-override-key4");
newCatalog.initialize("nessie", options);
return newCatalog;
}
Expand Down
6 changes: 6 additions & 0 deletions open-api/src/testFixtures/java/org/apache/iceberg/rest/RCKUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,12 @@ static RESTCatalog initCatalogClient(Map<String, String> properties) {
CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", "catalog-default-key1");
catalogProperties.putIfAbsent(
CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", "catalog-default-key2");
catalogProperties.putIfAbsent(
CatalogProperties.VIEW_DEFAULT_PREFIX + "key3", "catalog-default-key3");
catalogProperties.putIfAbsent(
CatalogProperties.VIEW_OVERRIDE_PREFIX + "key3", "catalog-override-key3");
catalogProperties.putIfAbsent(
CatalogProperties.VIEW_OVERRIDE_PREFIX + "key4", "catalog-override-key4");
catalogProperties.putIfAbsent(
CatalogProperties.TABLE_DEFAULT_PREFIX + "override-key3", "catalog-default-key3");
catalogProperties.putIfAbsent(
Expand Down