Skip to content

Commit e8a200d

Browse files
owlleg6owlleg6
authored
Merge pull request #18 from data-platform-hq/feat/refactor
feat: refactor
2 parents 9758d58 + badb031 commit e8a200d

File tree

3 files changed

+108
-135
lines changed

3 files changed

+108
-135
lines changed

‎README.md

Lines changed: 46 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -17,45 +17,56 @@ provider "databricks" {
1717
}
1818
1919
locals {
20-
metastore_id = "10000000-0000-0000-0000-0000000000"
21-
22-
metastore_grants = [
23-
{ principal = "<user1@email.com>", privileges = ["CREATE_CATALOG","CREATE_EXTERNAL_LOCATION"] },
24-
{ principal = "<user2@epam.com>", privileges = ["CREATE_SHARE", "CREATE_RECIPIENT", "CREATE_PROVIDER"] }
20+
catalog_config = [
21+
22+
# Catalog w/o grants
23+
{
24+
catalog_name = "catalog_with_no_grants"
25+
},
26+
27+
# Catalog with grants
28+
{
29+
catalog_name = "catalog_with_grants"
30+
catalog_grants = [
31+
{ principal = "account users", privileges = ["USE_CATALOG", "APPLY_TAG", "CREATE_SCHEMA", "SELECT"] }
32+
]
33+
},
34+
35+
# Catalog with grants and schemas
36+
{
37+
catalog_name = "catalog_with_schemas"
38+
catalog_grants = [{ principal = "account users", privileges = ["USE_CATALOG", "APPLY_TAG", "SELECT"] }]
39+
schema_configs = [
40+
{ schema_name = "schema_01" },
41+
{ schema_name = "schema_02" }
42+
]
43+
},
44+
45+
# Catalog with schemas where 'schema_01' and 'schema_02' have a default set of grants from 'schema_default_grants' parameter
46+
# and 'schema_03' has its own set of grants managed with 'schema_custom_grants' parameter
47+
{
48+
catalog_name = "catalog_custom_schema_grants"
49+
catalog_grants = [{ principal = "account users", privileges = ["USE_CATALOG", "APPLY_TAG"] }]
50+
schema_default_grants = [{ principal = "account users", privileges = ["CREATE_TABLE", "SELECT"] }]
51+
schema_configs = [
52+
{ schema_name = "schema_01" },
53+
{ schema_name = "schema_02" },
54+
{
55+
schema_name = "schema_03",
56+
schema_custom_grants = [
57+
{ principal = "account users", privileges = ["CREATE_VOLUME", "READ_VOLUME", "WRITE_VOLUME", "SELECT"] },
58+
]
59+
},
60+
]
61+
},
2562
]
26-
27-
catalog = {
28-
example_catalog = {
29-
catalog_grants = {
30-
"example@username.com" = ["USE_CATALOG", "USE_SCHEMA", "CREATE_SCHEMA", "CREATE_TABLE", "SELECT", "MODIFY"]
31-
}
32-
schema_name = ["raw", "refined", "data_product"]
33-
}
34-
}
35-
}
36-
37-
# Prerequisite module.
38-
# NOTE! It is required to assign Metastore to Workspace before creating Unity Catalog resources.
39-
module "metastore_assignment" {
40-
source = "data-platform-hq/metastore-assignment/databricks"
41-
version = "~> 1.0.0"
42-
43-
workspace_id = data.databricks_workspace.example.id
44-
metastore_id = local.metastore_id
45-
46-
providers = {
47-
databricks = databricks.workspace
48-
}
4963
}
5064
5165
module "unity_catalog" {
5266
source = "data-platform/unity-catalog/databricks"
53-
version = "~> 1.1.0"
67+
version = "~> 2.0.0"
5468
55-
env = "example"
56-
metastore_id = local.metastore_id
57-
metastore_grants = local.metastore_grants
58-
catalog = local.catalog
69+
catalog_config = local.catalog_config
5970
6071
providers = {
6172
databricks = databricks.workspace
@@ -88,7 +99,6 @@ No modules.
8899
|------|------|
89100
| [databricks_catalog.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/catalog) | resource |
90101
| [databricks_grants.catalog](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) | resource |
91-
| [databricks_grants.metastore](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) | resource |
92102
| [databricks_grants.schema](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/grants) | resource |
93103
| [databricks_schema.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/schema) | resource |
94104
| [databricks_workspace_binding.this](https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/workspace_binding) | resource |
@@ -97,10 +107,8 @@ No modules.
97107

98108
| Name | Description | Type | Default | Required |
99109
|------|-------------|------|---------|:--------:|
100-
| <a name="input_catalog"></a> [catalog](#input\_catalog) | Map of catalog name and its parameters | <pre>map(object({<br/> catalog_grants = optional(map(list(string)))<br/> catalog_owner = optional(string) # Username/groupname/sp application_id of the catalog owner.<br/> catalog_storage_root = optional(string) # Location in cloud storage where data for managed tables will be stored<br/> catalog_isolation_mode = optional(string, "OPEN") # Whether the catalog is accessible from all workspaces or a specific set of workspaces. Can be ISOLATED or OPEN.<br/> catalog_comment = optional(string) # User-supplied free-form text<br/> catalog_properties = optional(map(string)) # Extensible Catalog Tags.<br/> schema_name = optional(list(string)) # List of Schema names relative to parent catalog.<br/> schema_grants = optional(map(list(string)))<br/> schema_owner = optional(string) # Username/groupname/sp application_id of the schema owner.<br/> schema_comment = optional(string)<br/> schema_properties = optional(map(string))<br/> }))</pre> | `{}` | no |
101-
| <a name="input_isolated_unmanaged_catalog_bindings"></a> [isolated\_unmanaged\_catalog\_bindings](#input\_isolated\_unmanaged\_catalog\_bindings) | List of objects with parameters to configure Catalog Bindings | <pre>list(object({<br/> catalog_name = string # Name of ISOLATED catalog<br/> binding_type = optional(string, "BINDING_TYPE_READ_WRITE") # Binding mode. Possible values are BINDING_TYPE_READ_ONLY, BINDING_TYPE_READ_WRITE<br/> }))</pre> | `[]` | no |
102-
| <a name="input_metastore_grants"></a> [metastore\_grants](#input\_metastore\_grants) | Permissions to give on metastore to user, group or service principal | <pre>set(object({<br/> principal = string<br/> privileges = list(string)<br/> }))</pre> | `[]` | no |
103-
| <a name="input_metastore_id"></a> [metastore\_id](#input\_metastore\_id) | Unity Catalog Metastore Id that is located in separate environment. Provide this value to associate Databricks Workspace with target Metastore | `string` | n/a | yes |
110+
| <a name="input_catalog_config"></a> [catalog\_config](#input\_catalog\_config) | | <pre>list(object({<br/><br/> # Catalog config<br/> catalog_name = string<br/> catalog_owner = optional(string) # Username/groupname/sp application_id of the catalog owner.<br/> catalog_storage_root = optional(string) # Location in cloud storage where data for managed tables will be stored<br/> catalog_isolation_mode = optional(string, "OPEN") # Whether the catalog is accessible from all workspaces or a specific set of workspaces. Can be ISOLATED or OPEN.<br/> catalog_comment = optional(string) # User-supplied free-form text<br/> catalog_properties = optional(map(string)) # Extensible Catalog Tags.<br/> catalog_grants = optional(list(object({ # List of objects to set catalog permissions<br/> principal = string # Account level group name, user or service principal app ID<br/> privileges = list(string)<br/> })), [])<br/><br/> # Schemas<br/> schema_default_grants = optional(list(object({ # Sets default grants for each schema created by 'schema_configs' block w/o 'schema_custom_grants' parameter set<br/> principal = string # Account level group name, user or service principal app ID<br/> privileges = list(string)<br/> })), [])<br/><br/> schema_configs = optional(list(object({<br/> schema_name = string<br/> schema_owner = optional(string)<br/> schema_comment = optional(string)<br/> schema_properties = optional(map(string))<br/> schema_custom_grants = optional(list(object({ # Overwrites 'schema_default_grants'<br/> principal = string # Account level group name, user or service principal app ID<br/> privileges = list(string)<br/> })), [])<br/> })), [])<br/> }))</pre> | `[]` | no |
111+
| <a name="input_isolated_unmanaged_catalog_bindings"></a> [isolated\_unmanaged\_catalog\_bindings](#input\_isolated\_unmanaged\_catalog\_bindings) | | <pre>list(object({<br/> catalog_name = string # Name of ISOLATED catalog<br/> binding_type = optional(string, "BINDING_TYPE_READ_WRITE") # Binding mode. Possible values are BINDING_TYPE_READ_ONLY, BINDING_TYPE_READ_WRITE<br/> }))</pre> | `[]` | no |
104112
| <a name="input_workspace_id"></a> [workspace\_id](#input\_workspace\_id) | ID of the target workspace. | `string` | `null` | no |
105113

106114
## Outputs

‎main.tf

Lines changed: 29 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,21 @@
1-
# Metastore grants
21
locals {
3-
mapped_metastore_grants = {
4-
for object in var.metastore_grants : object.principal => object
5-
if object.principal != null
6-
}
7-
}
2+
catalogs_config_mapped = { for object in var.catalog_config : object.catalog_name => object }
83

9-
resource "databricks_grants" "metastore" {
10-
count = length(local.mapped_metastore_grants) != 0 ? 1 : 0
11-
12-
metastore = var.metastore_id
13-
dynamic "grant" {
14-
for_each = local.mapped_metastore_grants
15-
content {
16-
principal = grant.value.principal
17-
privileges = grant.value.privileges
18-
}
4+
schemas_config_mapped = {
5+
for config in flatten([for object in var.catalog_config : [for schema in object.schema_configs : {
6+
catalog = object.catalog_name
7+
schema = schema.schema_name
8+
schema_owner = schema.schema_owner
9+
schema_grants = try(coalescelist(schema.schema_custom_grants, object.schema_default_grants), [])
10+
}]]) : "${config.catalog}:${config.schema}" => config
1911
}
2012
}
2113

2214
# Catalog creation
2315
resource "databricks_catalog" "this" {
24-
for_each = var.catalog
16+
for_each = local.catalogs_config_mapped
2517

26-
metastore_id = var.metastore_id
27-
name = each.key
18+
name = each.value.catalog_name
2819
owner = each.value.catalog_owner
2920
storage_root = each.value.catalog_storage_root
3021
isolation_mode = each.value.catalog_isolation_mode
@@ -35,78 +26,49 @@ resource "databricks_catalog" "this" {
3526

3627
# Catalog grants
3728
resource "databricks_grants" "catalog" {
38-
for_each = {
39-
for name, params in var.catalog : name => params.catalog_grants
40-
if params.catalog_grants != null
41-
}
29+
for_each = { for k, v in local.catalogs_config_mapped : k => v if length(v.catalog_grants) != 0 }
4230

4331
catalog = databricks_catalog.this[each.key].name
32+
4433
dynamic "grant" {
45-
for_each = each.value
34+
for_each = each.value.catalog_grants
4635
content {
47-
principal = grant.key
48-
privileges = grant.value
36+
principal = grant.value.principal
37+
privileges = grant.value.privileges
4938
}
5039
}
5140
}
5241

5342
# Schema creation
54-
locals {
55-
schema = flatten([
56-
for catalog, params in var.catalog : [
57-
for schema in params.schema_name : {
58-
catalog = catalog,
59-
schema = schema,
60-
comment = lookup(params, "schema_comment", "default comment"),
61-
properties = lookup(params, "schema_properties", {})
62-
owner = lookup(params, "schema_owner", null)
63-
}
64-
] if params.schema_name != null
65-
])
66-
}
67-
6843
resource "databricks_schema" "this" {
69-
for_each = {
70-
for entry in local.schema : "${entry.catalog}.${entry.schema}" => entry
71-
}
44+
for_each = local.schemas_config_mapped
7245

7346
catalog_name = databricks_catalog.this[each.value.catalog].name
7447
name = each.value.schema
75-
owner = each.value.owner
76-
comment = each.value.comment
77-
properties = each.value.properties
48+
owner = each.value.schema_owner
49+
comment = each.value.schema_comment
50+
properties = each.value.schema_properties
7851
force_destroy = true
7952
}
8053

8154
# Schema grants
82-
locals {
83-
schema_grants = flatten([
84-
for catalog, params in var.catalog : [for schema in params.schema_name : [for principal in flatten(keys(params.schema_grants)) : {
85-
catalog = catalog,
86-
schema = schema,
87-
principal = principal,
88-
permission = flatten(values(params.schema_grants)),
89-
}]] if params.schema_grants != null
90-
])
91-
}
92-
9355
resource "databricks_grants" "schema" {
94-
for_each = {
95-
for entry in local.schema_grants : "${entry.catalog}.${entry.schema}.${entry.principal}" => entry
96-
}
56+
for_each = { for k, v in local.schemas_config_mapped : k => v if length(v.schema_grants) != 0 }
57+
58+
schema = databricks_schema.this[each.key].id
9759

98-
schema = databricks_schema.this["${each.value.catalog}.${each.value.schema}"].id
99-
grant {
100-
principal = each.value.principal
101-
privileges = each.value.permission
60+
dynamic "grant" {
61+
for_each = each.value.schema_grants
62+
content {
63+
principal = grant.value.principal
64+
privileges = grant.value.privileges
65+
}
10266
}
10367
}
10468

10569
# ISOLATED Catalogs binding
10670
resource "databricks_workspace_binding" "this" {
107-
for_each = {
108-
for object in var.isolated_unmanaged_catalog_bindings : object.catalog_name => object
109-
}
71+
for_each = { for object in var.isolated_unmanaged_catalog_bindings : object.catalog_name => object }
11072

11173
workspace_id = var.workspace_id
11274
securable_name = each.value.catalog_name

‎variables.tf

Lines changed: 33 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,50 @@
1-
variable "metastore_id" {
2-
type = string
3-
description = "Unity Catalog Metastore Id that is located in separate environment. Provide this value to associate Databricks Workspace with target Metastore"
4-
5-
validation {
6-
condition = length(var.metastore_id) == 36
7-
error_message = "Create Metastore or connect to existing one in Init Layer using Private Endpoint. In case Unity Catalog is not required, remove 'databricks_catalog' variable from tfvars file."
8-
}
9-
}
10-
11-
# Metastore grants
12-
variable "metastore_grants" {
13-
type = set(object({
14-
principal = string
15-
privileges = list(string)
16-
}))
17-
description = "Permissions to give on metastore to user, group or service principal"
18-
default = []
19-
}
1+
variable "catalog_config" {
2+
type = list(object({
203

21-
variable "catalog" {
22-
type = map(object({
23-
catalog_grants = optional(map(list(string)))
4+
# Catalog config
5+
catalog_name = string
246
catalog_owner = optional(string) # Username/groupname/sp application_id of the catalog owner.
257
catalog_storage_root = optional(string) # Location in cloud storage where data for managed tables will be stored
268
catalog_isolation_mode = optional(string, "OPEN") # Whether the catalog is accessible from all workspaces or a specific set of workspaces. Can be ISOLATED or OPEN.
279
catalog_comment = optional(string) # User-supplied free-form text
2810
catalog_properties = optional(map(string)) # Extensible Catalog Tags.
29-
schema_name = optional(list(string)) # List of Schema names relative to parent catalog.
30-
schema_grants = optional(map(list(string)))
31-
schema_owner = optional(string) # Username/groupname/sp application_id of the schema owner.
32-
schema_comment = optional(string)
33-
schema_properties = optional(map(string))
11+
catalog_grants = optional(list(object({ # List of objects to set catalog permissions
12+
principal = string # Account level group name, user or service principal app ID
13+
privileges = list(string)
14+
})), [])
15+
16+
# Schemas
17+
schema_default_grants = optional(list(object({ # Sets default grants for each schema created by 'schema_configs' block w/o 'schema_custom_grants' parameter set
18+
principal = string # Account level group name, user or service principal app ID
19+
privileges = list(string)
20+
})), [])
21+
22+
schema_configs = optional(list(object({
23+
schema_name = string
24+
schema_owner = optional(string)
25+
schema_comment = optional(string)
26+
schema_properties = optional(map(string))
27+
schema_custom_grants = optional(list(object({ # Overwrites 'schema_default_grants'
28+
principal = string # Account level group name, user or service principal app ID
29+
privileges = list(string)
30+
})), [])
31+
})), [])
3432
}))
35-
description = "Map of catalog name and its parameters"
36-
default = {}
33+
description = <<DESCRIPTION
34+
35+
36+
DESCRIPTION
37+
default = []
3738
}
3839

3940
variable "isolated_unmanaged_catalog_bindings" {
4041
type = list(object({
4142
catalog_name = string # Name of ISOLATED catalog
4243
binding_type = optional(string, "BINDING_TYPE_READ_WRITE") # Binding mode. Possible values are BINDING_TYPE_READ_ONLY, BINDING_TYPE_READ_WRITE
4344
}))
44-
description = "List of objects with parameters to configure Catalog Bindings"
45+
description = <<DESCRIPTION
46+
47+
DESCRIPTION
4548
default = []
4649
}
4750

0 commit comments

Comments
 (0)