Skip to content

Commit bcbd38b

Browse files
mkananizadehmkananizadeh
authored
mkananizadeh:Added Data Portal and updated Terraform plan (#36)
1 parent 5406f4f commit bcbd38b

27 files changed

+2594
-2098
lines changed

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -366,6 +366,8 @@ notes
366366
.env
367367
key.json
368368
resources.json
369+
team_customer_support.txt
370+
team_marketing_analytics.txt
369371

370372
# Storage
371373
**/.DS_Store

‎README.md

+105-40
Large diffs are not rendered by default.

‎confluent/create-business-metadata.sh

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
METADATA_FILE=$1
2+
3+
source ../.env
4+
5+
curl -u $CCLOUD_SCHEMA_REGISTRY_API_KEY:$CCLOUD_SCHEMA_REGISTRY_API_SECRET -X POST -H "Content-Type: application/json" \
6+
--data @$METADATA_FILE "{$CCLOUD_SCHEMA_REGISTRY_URL}/catalog/v1/types/businessmetadatadefs" | jq .

‎confluent/create-tag.sh

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
TAGFILE=$1
2+
3+
source ../.env
4+
5+
curl -u $CCLOUD_SCHEMA_REGISTRY_API_KEY:$CCLOUD_SCHEMA_REGISTRY_API_SECRET \
6+
--header 'Content-Type: application/json' \
7+
--data @$TAGFILE \
8+
--url "{$CCLOUD_SCHEMA_REGISTRY_URL}/catalog/v1/types/tagdefs" | jq .

‎create_env.sh renamed to ‎confluent/create_env.sh

+9-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
#!/bin/bash
22

3-
accounts_file=".accounts"
4-
env_file=".env"
3+
current_dir=$(pwd)
4+
parent_dir=$(dirname "$current_dir")
5+
6+
accounts_file="${parent_dir}/.accounts"
7+
env_file="${parent_dir}/.env"
58

69
# Check if .accounts file exists
710
if [ ! -f "$accounts_file" ]; then
@@ -15,6 +18,10 @@ CCLOUD_API_KEY=api-key
1518
CCLOUD_API_SECRET=api-secret
1619
CCLOUD_BOOTSTRAP_ENDPOINT=kafka-cluster-endpoint
1720
21+
CCLOUD_SCHEMA_REGISTRY_API_KEY=sr-key
22+
CCLOUD_SCHEMA_REGISTRY_API_SECRET=sr-secret
23+
CCLOUD_SCHEMA_REGISTRY_URL=sr-cluster-endpoint
24+
1825
ORACLE_USERNAME=admin
1926
ORACLE_PASSWORD=demo-cdc-c0nflu3nt!
2027
ORACLE_ENDPOINT=oracle-endpoint

‎confluent/dataprod-tag.json

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[ { "entityTypes" : [ "cf_entity" ],"name" : "DataProduct","description" : "Enriched customer data."} ]

‎confluent/dlq-tag.json

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[ { "entityTypes" : [ "cf_entity" ],"name" : "RAW","description" : "Real-time raw data streamed from a data source."} ]

‎ksqldb_queries.sql renamed to ‎confluent/ksqldb_queries.sql

File renamed without changes.

‎confluent/pii-tag.json

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[ { "entityTypes" : [ "cf_entity" ],"name" : "Sensitive","description" : "Confidential information that must be kept safe and out of reach from all outsiders unless they have permission to access it."} ]

‎confluent/private-tag.json

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[ { "entityTypes" : [ "cf_entity" ],"name" : "Private","description" : "Private data that are not to be made public."} ]

‎confluent/raw-tag.json

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[ { "entityTypes" : [ "cf_entity" ],"name" : "RAW","description" : "Real-time raw data streamed from a data source."} ]

‎confluent/sensitive-tag.json

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[ { "entityTypes" : [ "cf_entity" ],"name" : "Sensitive","description" : "Confidential information that must be kept safe and out of reach from all outsiders unless they have permission to access it."} ]

‎setup.sh renamed to ‎confluent/setup.sh

+48-11
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
#!/bin/bash
22

33
sleep_time=2
4-
env_file=".env"
4+
current_dir=$(pwd)
5+
parent_dir=$(dirname "$current_dir")
6+
7+
env_file="${parent_dir}/.env"
8+
resources_file="${parent_dir}/resources.json"
59

610
# Use confluent environment
711
confluent login --save
@@ -21,7 +25,7 @@ export CCLOUD_BOOTSTRAP_ENDPOINT=$(confluent kafka cluster describe -o json | jq
2125
STRIPPED_CCLOUD_BOOTSTRAP_ENDPOINT=$(echo $CCLOUD_BOOTSTRAP_ENDPOINT | sed 's/SASL_SSL:\/\///')
2226

2327
# use sed to replace kafka-cluster-endpoint with the replacement string
24-
sed -i .bak "s/kafka-cluster-endpoint/$STRIPPED_CCLOUD_BOOTSTRAP_ENDPOINT/g" $env_file
28+
sed -i .bak "s/kafka-cluster-endpoint/$STRIPPED_CCLOUD_BOOTSTRAP_ENDPOINT/g" "$env_file"
2529
echo "Added Kafka cluster endpoint to $env_file"
2630
sleep $sleep_time
2731

@@ -33,15 +37,50 @@ kafka_api_secret=$(echo $CREDENTIALS | jq -r '.api_secret')
3337
sleep $sleep_time
3438

3539
# use sed to replace all instances of $kafka_api_key with the replacement string
36-
sed -i .bak "s^api-key^\"$kafka_api_key\"^g" $env_file
37-
sed -i .bak "s^api-secret^\"$kafka_api_secret\"^g" $env_file
40+
sed -i .bak "s^api-key^\"$kafka_api_key\"^g" "$env_file"
41+
sed -i .bak "s^api-secret^\"$kafka_api_secret\"^g" "$env_file"
3842
echo "Added Kafka API key and secret to $env_file"
3943

4044
sleep $sleep_time
4145

46+
# Get schema registry info
47+
export CCLOUD_SCHEMA_REGISTRY_ID=$(confluent sr cluster describe -o json | jq -r .cluster_id)
48+
export CCLOUD_SCHEMA_REGISTRY_ENDPOINT=$(confluent sr cluster describe -o json | jq -r .endpoint_url)
49+
50+
echo ""
51+
echo "Creating schema registry API key"
52+
SR_CREDENTIALS=$(confluent api-key create --resource $CCLOUD_SCHEMA_REGISTRY_ID --description "demo-change-data-capture" -o json)
53+
sr_api_key=$(echo $SR_CREDENTIALS | jq -r '.api_key')
54+
sr_api_secret=$(echo $SR_CREDENTIALS | jq -r '.api_secret')
55+
sleep $sleep_time
56+
57+
# use sed to replace all instances of $sr_api_key and $sr_api_secret with the replacement string
58+
sed -i .bak "s^sr-key^\"$sr_api_key\"^g" "$env_file"
59+
sed -i .bak "s^sr-secret^\"$sr_api_secret\"^g" "$env_file"
60+
sed -i .bak "s^sr-cluster-endpoint^$CCLOUD_SCHEMA_REGISTRY_ENDPOINT^g" "$env_file"
61+
sleep $sleep_time
62+
63+
# source the $env_file file
64+
source "$env_file"
65+
66+
# Create tags for topics
67+
echo ""
68+
echo "Creating tags"
69+
./create-tag.sh ./pii-tag.json
70+
./create-tag.sh ./private-tag.json
71+
./create-tag.sh ./sensitive-tag.json
72+
./create-tag.sh ./dataprod-tag.json
73+
./create-tag.sh ./raw-tag.json
74+
./create-tag.sh ./dlq-tag.json
75+
76+
# Create business metadata for topics
77+
echo ""
78+
echo "Creating business metadata"
79+
./create-business-metadata.sh ./team.txt
80+
4281
# Read values from resources.json and update the $env_file file.
4382
# These resources are created by Terraform
44-
json=$(cat resources.json)
83+
json=$(cat "$resources_file")
4584

4685
oracle_endpoint=$(echo "$json" | jq -r '.oracle_endpoint.value.address')
4786
postgres_products=$(echo "$json" | jq -r '.postgres_instance_products_public_endpoint.value')
@@ -53,14 +92,12 @@ redshift_endpoint=$(echo "$json" | jq -r '.redshift_endpoint.value')
5392
redshift_address=$(echo $redshift_endpoint | sed 's/:5439//')
5493

5594
# Updating the $env_file file with sed command
56-
sed -i .bak "s^oracle-endpoint^$oracle_endpoint^g" $env_file
57-
sed -i .bak "s^postgres-products^$postgres_products^g" $env_file
58-
sed -i .bak "s^snowflake-private-key^\"$snowflake_svc_private_key\"^g" $env_file
59-
sed -i .bak "s^redshift-address^$redshift_address^g" $env_file
95+
sed -i .bak "s^oracle-endpoint^$oracle_endpoint^g" "$env_file"
96+
sed -i .bak "s^postgres-products^$postgres_products^g" "$env_file"
97+
sed -i .bak "s^snowflake-private-key^\"$snowflake_svc_private_key\"^g" "$env_file"
98+
sed -i .bak "s^redshift-address^$redshift_address^g" "$env_file"
6099

61100
echo "Added Oracle endpoint to $env_file"
62101
echo "Added PostgreSQL endpoint to $env_file"
63102
echo "Added Snowflake private key to $env_file"
64103
echo "Added Amazon Redshift address to $env_file"
65-
66-
# sleep $sleep_time

‎confluent/team.txt

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
[ {
2+
"name" : "Domain",
3+
"description" : "Enriched data at Acme Corp.",
4+
"attributeDefs" : [
5+
{ "name" : "Team_owner", "isOptional" : "true", "cardinality" : "SINGLE", "typeName" : "string", "options": {"maxStrLength": 30, "applicableEntityTypes": "[\"cf_entity\"]"} },
6+
{ "name" : "Slack_contact", "isOptional" : "true", "cardinality" : "SINGLE", "typeName" : "string", "options": {"maxStrLength": 30, "applicableEntityTypes": "[\"cf_entity\"]"} },
7+
{ "name" : "Name", "isOptional" : "true", "cardinality" : "SINGLE", "typeName" : "string", "options": {"maxStrLength": 30, "applicableEntityTypes": "[\"cf_entity\"]"} }
8+
]
9+
} ]

‎teardown_connectors.sh renamed to ‎confluent/teardown_connectors.sh

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
#!/bin/bash
22

33
# Source the .env file
4-
source .env
4+
current_dir=$(pwd)
5+
parent_dir=$(dirname "$current_dir")
6+
7+
env_file="${parent_dir}/.env"
58

69
# Use confluent environment
710
confluent login --save

0 commit comments

Comments
 (0)