diff --git a/.claude/settings.local.json b/.claude/settings.local.json index b0de50f..c16c249 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -11,7 +11,9 @@ "Bash(cargo tree:*)", "Bash(where llvm-config:*)", "Read(//c/msys64/mingw64/bin/**)", - "Read(//c/msys64/clang64/bin/**)" + "Read(//c/msys64/clang64/bin/**)", + "Bash(cargo doc:*)", + "Bash(grep -r impl.*Read.*for.*Response c:/LocalGitRepos/stackql/stackql-deploy-rs/target/doc/reqwest/blocking/struct.Response.html)" ] } } diff --git a/README.md b/README.md index e365d6a..303156f 100644 --- a/README.md +++ b/README.md @@ -68,13 +68,13 @@ Resource `.iql` files define mutation and check queries using SQL anchors: ```sql /*+ create */ INSERT INTO azure.resources.resource_groups( - resourceGroupName, subscriptionId, data__location + resourceGroupName, subscriptionId, location ) SELECT '{{ resource_group_name }}', '{{ subscription_id }}', '{{ location }}' /*+ update */ UPDATE azure.resources.resource_groups -SET data__location = '{{ location }}' +SET location = '{{ location }}' WHERE resourceGroupName = '{{ resource_group_name }}' AND subscriptionId = '{{ subscription_id }}' diff --git a/examples/aws/aws-stack/resources/example_inet_gateway.iql b/examples/aws/aws-stack/resources/example_inet_gateway.iql index 473b4c0..5bd42b2 100644 --- a/examples/aws/aws-stack/resources/example_inet_gateway.iql +++ b/examples/aws/aws-stack/resources/example_inet_gateway.iql @@ -48,5 +48,5 @@ AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}' /*+ delete */ DELETE FROM aws.ec2.internet_gateways -WHERE data__Identifier = '{{ internet_gateway_id }}' +WHERE Identifier = '{{ internet_gateway_id }}' AND region = '{{ region }}'; diff --git a/examples/aws/aws-stack/resources/example_inet_gw_attachment.iql b/examples/aws/aws-stack/resources/example_inet_gw_attachment.iql index 28138a8..b6e1def 100644 --- a/examples/aws/aws-stack/resources/example_inet_gw_attachment.iql +++ b/examples/aws/aws-stack/resources/example_inet_gw_attachment.iql @@ -35,5 +35,5 @@ AND vpc_id = '{{ vpc_id }}' /*+ delete */ DELETE FROM aws.ec2.vpc_gateway_attachments -WHERE data__Identifier = 'IGW|{{ vpc_id }}' +WHERE Identifier = 'IGW|{{ vpc_id }}' AND region = '{{ region }}'; \ No newline at end of file diff --git a/examples/aws/aws-stack/resources/example_inet_route.iql b/examples/aws/aws-stack/resources/example_inet_route.iql index 105b06b..d9454fb 100644 --- a/examples/aws/aws-stack/resources/example_inet_route.iql +++ b/examples/aws/aws-stack/resources/example_inet_route.iql @@ -1,10 +1,10 @@ /*+ exists */ SELECT COUNT(*) as count FROM ( -SELECT data__Identifier +SELECT Identifier FROM aws.ec2.routes WHERE region = '{{ region }}' -AND data__Identifier = '{{ route_table_id }}|0.0.0.0/0' +AND Identifier = '{{ route_table_id }}|0.0.0.0/0' ) t; /*+ create */ @@ -23,19 +23,19 @@ SELECT /*+ statecheck, retries=5, retry_delay=5 */ SELECT COUNT(*) as count FROM ( -SELECT data__Identifier +SELECT Identifier FROM aws.ec2.routes WHERE region = '{{ region }}' -AND data__Identifier = '{{ route_table_id }}|0.0.0.0/0' +AND Identifier = '{{ route_table_id }}|0.0.0.0/0' ) t; /*+ exports */ -SELECT data__Identifier as inet_route_indentifer +SELECT Identifier as inet_route_indentifer FROM aws.ec2.routes WHERE region = '{{ region }}' -AND data__Identifier = '{{ route_table_id }}|0.0.0.0/0'; +AND Identifier = '{{ route_table_id }}|0.0.0.0/0'; /*+ delete */ DELETE FROM aws.ec2.routes -WHERE data__Identifier = '{{ inet_route_indentifer }}' +WHERE Identifier = '{{ inet_route_indentifer }}' AND region = '{{ region }}'; \ No newline at end of file diff --git a/examples/aws/aws-stack/resources/example_route_table.iql b/examples/aws/aws-stack/resources/example_route_table.iql index 6a56af8..4333d18 100644 --- a/examples/aws/aws-stack/resources/example_route_table.iql +++ b/examples/aws/aws-stack/resources/example_route_table.iql @@ -53,5 +53,5 @@ AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}' /*+ delete */ DELETE FROM aws.ec2.route_tables -WHERE data__Identifier = '{{ route_table_id }}' +WHERE Identifier = '{{ route_table_id }}' AND region = '{{ region }}'; \ No newline at end of file diff --git a/examples/aws/aws-stack/resources/example_security_group.iql b/examples/aws/aws-stack/resources/example_security_group.iql index 485a761..0f57ce9 100644 --- a/examples/aws/aws-stack/resources/example_security_group.iql +++ b/examples/aws/aws-stack/resources/example_security_group.iql @@ -68,5 +68,5 @@ AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}' /*+ delete */ DELETE FROM aws.ec2.security_groups -WHERE data__Identifier = '{{ security_group_id }}' +WHERE Identifier = '{{ security_group_id }}' AND region = '{{ region }}'; \ No newline at end of file diff --git a/examples/aws/aws-stack/resources/example_subnet.iql b/examples/aws/aws-stack/resources/example_subnet.iql index 5f62cb0..d03699e 100644 --- a/examples/aws/aws-stack/resources/example_subnet.iql +++ b/examples/aws/aws-stack/resources/example_subnet.iql @@ -62,5 +62,5 @@ WHERE cidr_block = '{{ subnet_cidr_block }}'; /*+ delete */ DELETE FROM aws.ec2.subnets -WHERE data__Identifier = '{{ subnet_id }}' +WHERE Identifier = '{{ subnet_id }}' AND region = '{{ region }}'; \ No newline at end of file diff --git a/examples/aws/aws-stack/resources/example_subnet_rt_assn.iql b/examples/aws/aws-stack/resources/example_subnet_rt_assn.iql index 58c80f4..d0c81dc 100644 --- a/examples/aws/aws-stack/resources/example_subnet_rt_assn.iql +++ b/examples/aws/aws-stack/resources/example_subnet_rt_assn.iql @@ -38,5 +38,5 @@ AND subnet_id = '{{ subnet_id }}'; /*+ delete */ DELETE FROM aws.ec2.subnet_route_table_associations -WHERE data__Identifier = '{{ route_table_assn_id }}' +WHERE Identifier = '{{ route_table_assn_id }}' AND region = '{{ region }}'; \ No newline at end of file diff --git a/examples/aws/aws-stack/resources/example_vpc.iql b/examples/aws/aws-stack/resources/example_vpc.iql index 35b2733..63d52ce 100644 --- a/examples/aws/aws-stack/resources/example_vpc.iql +++ b/examples/aws/aws-stack/resources/example_vpc.iql @@ -59,5 +59,5 @@ AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}' /*+ delete */ DELETE FROM aws.ec2.vpcs -WHERE data__Identifier = '{{ vpc_id }}' +WHERE Identifier = '{{ vpc_id }}' AND region = '{{ region }}'; \ No newline at end of file diff --git a/examples/aws/aws-stack/resources/example_web_server.iql b/examples/aws/aws-stack/resources/example_web_server.iql index e479969..83de535 100644 --- a/examples/aws/aws-stack/resources/example_web_server.iql +++ b/examples/aws/aws-stack/resources/example_web_server.iql @@ -67,5 +67,5 @@ AND json_extract(tags, '$.Name') = '{{ instance_name }}' /*+ delete */ DELETE FROM aws.ec2.instances -WHERE data__Identifier = '{{ instance_id }}' +WHERE Identifier = '{{ instance_id }}' AND region = '{{ region }}'; \ No newline at end of file diff --git a/examples/aws/aws-stack/stackql_manifest.yml b/examples/aws/aws-stack/stackql_manifest.yml index 19f6251..7b0673a 100644 --- a/examples/aws/aws-stack/stackql_manifest.yml +++ b/examples/aws/aws-stack/stackql_manifest.yml @@ -1,23 +1,20 @@ -# -# aws starter project manifest file, add and update values as needed -# version: 1 -name: "aws-stack" -description: description for "aws-stack" +name: "aws-vpc-webserver" +description: Provisions a complete AWS networking stack (VPC, subnet, internet gateway, route table, security group) with an Apache web server EC2 instance. providers: - - aws + - awscc globals: - name: region description: aws region value: "{{ AWS_REGION }}" - name: global_tags value: - - Key: Provisioner - Value: stackql - - Key: StackName + - Key: 'stackql:stack-name' Value: "{{ stack_name }}" - - Key: StackEnv + - Key: 'stackql:stack-env' Value: "{{ stack_env }}" + - Key: 'stackql:resource-name' + Value: "{{ resource_name }}" resources: - name: example_vpc props: diff --git a/examples/aws/patch-doc-test/README.md b/examples/aws/patch-doc-test/README.md deleted file mode 100644 index 0b72a5a..0000000 --- a/examples/aws/patch-doc-test/README.md +++ /dev/null @@ -1,80 +0,0 @@ -# `stackql-deploy` starter project for `aws` - -> for starter projects using other providers, try `stackql-deploy patch-doc-test --provider=azure` or `stackql-deploy patch-doc-test --provider=google` - -see the following links for more information on `stackql`, `stackql-deploy` and the `aws` provider: - -- [`aws` provider docs](https://stackql.io/registry/aws) -- [`stackql`](https://github.com/stackql/stackql) -- [`stackql-deploy` PyPI home page](https://pypi.org/project/stackql-deploy/) -- [`stackql-deploy` GitHub repo](https://github.com/stackql/stackql-deploy) - -## Overview - -__`stackql-deploy`__ is a stateless, declarative, SQL driven Infrastructure-as-Code (IaC) framework. There is no state file required as the current state is assessed for each resource at runtime. __`stackql-deploy`__ is capable of provisioning, deprovisioning and testing a stack which can include resources across different providers, like a stack spanning `aws` and `azure` for example. - -## Prerequisites - -This example requires `stackql-deploy` to be installed using __`pip install stackql-deploy`__. The host used to run `stackql-deploy` needs the necessary environment variables set to authenticate to your specific provider, in the case of the `aws` provider, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and optionally `AWS_SESSION_TOKEN` must be set, for more information on authentication to `aws` see the [`aws` provider documentation](https://aws.stackql.io/providers/aws). - -> __Note for macOS users__ -> to install `stackql-deploy` in a virtual environment (which may be necessary on __macOS__), use the following: -> ```bash -> python3 -m venv myenv -> source myenv/bin/activate -> pip install stackql-deploy -> ``` - -## Usage - -Adjust the values in the [__`stackql_manifest.yml`__](stackql_manifest.yml) file if desired. The [__`stackql_manifest.yml`__](stackql_manifest.yml) file contains resource configuration variables to support multiple deployment environments, these will be used for `stackql` queries in the `resources` folder. - -The syntax for the `stackql-deploy` command is as follows: - -```bash -stackql-deploy { build | test | teardown } { stack-directory } { deployment environment} [ optional flags ] -``` - -### Deploying a stack - -For example, to deploy the stack named patch-doc-test to an environment labeled `sit`, run the following: - -```bash -stackql-deploy build \ -examples/aws/patch-doc-test \ -sit \ --e AWS_REGION=ap-southeast-2 \ ---show-queries -``` - -Use the `--dry-run` flag to view the queries to be run without actually running them, for example: - -```bash -stackql-deploy build \ -examples/aws/patch-doc-test \ -sit \ --e AWS_REGION=ap-southeast-2 \ ---dry-run -``` - -### Testing a stack - -To test a stack to ensure that all resources are present and in the desired state, run the following (in our `sit` deployment example): - -```bash -stackql-deploy test \ -examples/aws/patch-doc-test \ -sit \ --e AWS_REGION=ap-southeast-2 -``` - -### Tearing down a stack - -To destroy or deprovision all resources in a stack for our `sit` deployment example, run the following: - -```bash -stackql-deploy teardown \ -examples/aws/patch-doc-test \ -sit \ --e AWS_REGION=ap-southeast-2 -``` \ No newline at end of file diff --git a/examples/aws/patch-doc-test/resources/bucket1.iql b/examples/aws/patch-doc-test/resources/bucket1.iql deleted file mode 100644 index b11970b..0000000 --- a/examples/aws/patch-doc-test/resources/bucket1.iql +++ /dev/null @@ -1,54 +0,0 @@ -/*+ exists */ -SELECT -COUNT(*) as count -FROM aws.s3.buckets -WHERE region = '{{ region }}' AND data__Identifier = '{{ bucket1_name }}' - -/*+ create */ -INSERT INTO aws.s3.buckets ( - BucketName, - VersioningConfiguration, - Tags, - region -) -SELECT - '{{ bucket1_name }}', - '{{ bucket1_versioning_config }}', - '{{ bucket1_tags }}', - '{{ region }}' - -/*+ statecheck, retries=2, retry_delay=1 */ -SELECT COUNT(*) as count FROM -( -SELECT -JSON_EQUAL(versioning_configuration, '{{ bucket1_versioning_config }}') as test_versioning_config -FROM aws.s3.buckets -WHERE region = '{{ region }}' -AND data__Identifier = '{{ bucket1_name }}' -) t -WHERE test_versioning_config = 1; - -/*+ exports, retries=2, retry_delay=1 */ -SELECT bucket_name as bucket1_name, arn as bucket1_arn FROM -( -SELECT -bucket_name, -arn -FROM aws.s3.buckets -WHERE region = '{{ region }}' -AND data__Identifier = '{{ bucket1_name }}' -) t - -/*+ update */ -update aws.s3.buckets -set data__PatchDocument = string('{{ { - "VersioningConfiguration": bucket1_versioning_config, - "Tags": bucket1_tags - } | generate_patch_document }}') -WHERE region = '{{ region }}' -AND data__Identifier = '{{ bucket1_name }}'; - -/*+ delete */ -DELETE FROM aws.s3.buckets -WHERE data__Identifier = '{{ bucket1_name }}' -AND region = '{{ region }}' diff --git a/examples/aws/patch-doc-test/stackql_manifest.yml b/examples/aws/patch-doc-test/stackql_manifest.yml deleted file mode 100644 index 0244891..0000000 --- a/examples/aws/patch-doc-test/stackql_manifest.yml +++ /dev/null @@ -1,34 +0,0 @@ -version: 1 -name: "patch-doc-test" -description: description for "patch-doc-test" -providers: - - aws -globals: - - name: region - description: aws region - value: "{{ AWS_REGION }}" - - name: global_tags - value: - - Key: Provisioner - Value: stackql - - Key: StackName - Value: "{{ stack_name }}" - - Key: StackEnv - Value: "{{ stack_env }}" -resources: - - name: bucket1 - props: - - name: bucket1_name - value: "{{ stack_name }}-{{ stack_env }}-bucket1" - - name: bucket1_versioning_config - value: - Status: Enabled - - name: bucket1_tags - merge: - - global_tags - value: - - Key: Name - Value: "{{ stack_name }}-{{ stack_env }}-bucket1" - exports: - - bucket1_name - - bucket1_arn diff --git a/examples/azure/azure-stack/resources/example_nic.iql b/examples/azure/azure-stack/resources/example_nic.iql index 27be6fc..6b3f1fa 100644 --- a/examples/azure/azure-stack/resources/example_nic.iql +++ b/examples/azure/azure-stack/resources/example_nic.iql @@ -3,9 +3,9 @@ INSERT INTO azure.network.interfaces( networkInterfaceName, resourceGroupName, subscriptionId, - data__location, - data__properties, - data__tags + location, + properties, + tags ) SELECT '{{ nic_name }}', diff --git a/examples/azure/azure-stack/resources/example_nsg.iql b/examples/azure/azure-stack/resources/example_nsg.iql index 5d37386..48a24ce 100644 --- a/examples/azure/azure-stack/resources/example_nsg.iql +++ b/examples/azure/azure-stack/resources/example_nsg.iql @@ -3,9 +3,9 @@ INSERT INTO azure.network.security_groups( networkSecurityGroupName, resourceGroupName, subscriptionId, - data__location, - data__properties, - data__tags + location, + properties, + tags ) SELECT '{{ nsg_name }}', diff --git a/examples/azure/azure-stack/resources/example_public_ip.iql b/examples/azure/azure-stack/resources/example_public_ip.iql index 5636482..b244378 100644 --- a/examples/azure/azure-stack/resources/example_public_ip.iql +++ b/examples/azure/azure-stack/resources/example_public_ip.iql @@ -3,9 +3,9 @@ INSERT INTO azure.network.public_ip_addresses( publicIpAddressName, resourceGroupName, subscriptionId, - data__location, - data__properties, - data__tags + location, + properties, + tags ) SELECT '{{ public_ip_name }}', diff --git a/examples/azure/azure-stack/resources/example_resource_group.iql b/examples/azure/azure-stack/resources/example_resource_group.iql index dc9c4b6..265e452 100644 --- a/examples/azure/azure-stack/resources/example_resource_group.iql +++ b/examples/azure/azure-stack/resources/example_resource_group.iql @@ -7,8 +7,8 @@ AND resourceGroupName = '{{ resource_group_name }}' INSERT INTO azure.resources.resource_groups( resourceGroupName, subscriptionId, - data__location, - data__tags + location, + tags ) SELECT '{{ resource_group_name }}', diff --git a/examples/azure/azure-stack/resources/example_subnet.iql b/examples/azure/azure-stack/resources/example_subnet.iql index fffb317..1be82b3 100644 --- a/examples/azure/azure-stack/resources/example_subnet.iql +++ b/examples/azure/azure-stack/resources/example_subnet.iql @@ -4,7 +4,7 @@ INSERT INTO azure.network.subnets( virtualNetworkName, resourceGroupName, subscriptionId, - data__properties + properties ) SELECT '{{ subnet_name }}', diff --git a/examples/azure/azure-stack/resources/example_vm_ext.iql b/examples/azure/azure-stack/resources/example_vm_ext.iql index 6291d15..42b9942 100644 --- a/examples/azure/azure-stack/resources/example_vm_ext.iql +++ b/examples/azure/azure-stack/resources/example_vm_ext.iql @@ -4,9 +4,9 @@ INSERT INTO azure.compute.virtual_machine_extensions( subscriptionId, vmExtensionName, vmName, - data__location, - data__properties, - data__tags + location, + properties, + tags ) SELECT '{{ resource_group_name }}', diff --git a/examples/azure/azure-stack/resources/example_vnet.iql b/examples/azure/azure-stack/resources/example_vnet.iql index 55fc558..152414b 100644 --- a/examples/azure/azure-stack/resources/example_vnet.iql +++ b/examples/azure/azure-stack/resources/example_vnet.iql @@ -3,9 +3,9 @@ INSERT INTO azure.network.virtual_networks( virtualNetworkName, resourceGroupName, subscriptionId, - data__location, - data__properties, - data__tags + location, + properties, + tags ) SELECT '{{ vnet_name }}', diff --git a/examples/azure/azure-stack/resources/example_web_server.iql b/examples/azure/azure-stack/resources/example_web_server.iql index a069441..f53bdfe 100644 --- a/examples/azure/azure-stack/resources/example_web_server.iql +++ b/examples/azure/azure-stack/resources/example_web_server.iql @@ -3,9 +3,9 @@ INSERT INTO azure.compute.virtual_machines( resourceGroupName, subscriptionId, vmName, - data__location, - data__properties, - data__tags + location, + properties, + tags ) SELECT '{{ resource_group_name }}', diff --git a/examples/confluent/cmd-specific-auth/README.md b/examples/confluent/cmd-specific-auth/README.md deleted file mode 100644 index e56f49d..0000000 --- a/examples/confluent/cmd-specific-auth/README.md +++ /dev/null @@ -1,63 +0,0 @@ -# `stackql-deploy` starter project for `aws` - -> for starter projects using other providers, try `stackql-deploy cmd-specific-auth --provider=azure` or `stackql-deploy cmd-specific-auth --provider=google` - -see the following links for more information on `stackql`, `stackql-deploy` and the `aws` provider: - -- [`aws` provider docs](https://stackql.io/registry/aws) -- [`stackql`](https://github.com/stackql/stackql) -- [`stackql-deploy` PyPI home page](https://pypi.org/project/stackql-deploy/) -- [`stackql-deploy` GitHub repo](https://github.com/stackql/stackql-deploy) - -## Overview - -__`stackql-deploy`__ is a stateless, declarative, SQL driven Infrastructure-as-Code (IaC) framework. There is no state file required as the current state is assessed for each resource at runtime. __`stackql-deploy`__ is capable of provisioning, deprovisioning and testing a stack which can include resources across different providers, like a stack spanning `aws` and `azure` for example. - -## Prerequisites - -This example requires `stackql-deploy` to be installed using __`pip install stackql-deploy`__. The host used to run `stackql-deploy` needs the necessary environment variables set to authenticate to your specific provider, in the case of the `aws` provider, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and optionally `AWS_SESSION_TOKEN` must be set, for more information on authentication to `aws` see the [`aws` provider documentation](https://aws.stackql.io/providers/aws). - -## Usage - -Adjust the values in the [__`stackql_manifest.yml`__](stackql_manifest.yml) file if desired. The [__`stackql_manifest.yml`__](stackql_manifest.yml) file contains resource configuration variables to support multiple deployment environments, these will be used for `stackql` queries in the `resources` folder. - -The syntax for the `stackql-deploy` command is as follows: - -```bash -stackql-deploy { build | test | teardown } { stack-directory } { deployment environment} [ optional flags ] -``` - -### Deploying a stack - -For example, to deploy the stack named cmd-specific-auth to an environment labeled `sit`, run the following: - -```bash -stackql-deploy build cmd-specific-auth sit \ --e AWS_REGION=ap-southeast-2 -``` - -Use the `--dry-run` flag to view the queries to be run without actually running them, for example: - -```bash -stackql-deploy build cmd-specific-auth sit \ --e AWS_REGION=ap-southeast-2 \ ---dry-run -``` - -### Testing a stack - -To test a stack to ensure that all resources are present and in the desired state, run the following (in our `sit` deployment example): - -```bash -stackql-deploy test cmd-specific-auth sit \ --e AWS_REGION=ap-southeast-2 -``` - -### Tearing down a stack - -To destroy or deprovision all resources in a stack for our `sit` deployment example, run the following: - -```bash -stackql-deploy teardown cmd-specific-auth sit \ --e AWS_REGION=ap-southeast-2 -``` \ No newline at end of file diff --git a/examples/confluent/cmd-specific-auth/resources/example_vpc.iql b/examples/confluent/cmd-specific-auth/resources/example_vpc.iql deleted file mode 100644 index 463dbc1..0000000 --- a/examples/confluent/cmd-specific-auth/resources/example_vpc.iql +++ /dev/null @@ -1,67 +0,0 @@ -/* defines the provisioning and deprovisioning commands -used to create, update or delete the resource -replace queries with your queries */ - -/*+ exists */ -SELECT COUNT(*) as count FROM -( -SELECT vpc_id, -json_group_object(tag_key, tag_value) as tags -FROM aws.ec2.vpc_tags -WHERE region = '{{ region }}' -AND cidr_block = '{{ vpc_cidr_block }}' -GROUP BY vpc_id -HAVING json_extract(tags, '$.Provisioner') = 'stackql' -AND json_extract(tags, '$.StackName') = '{{ stack_name }}' -AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}' -) t; - -/*+ create */ -INSERT INTO aws.ec2.vpcs ( - CidrBlock, - Tags, - EnableDnsSupport, - EnableDnsHostnames, - region -) -SELECT - '{{ vpc_cidr_block }}', - '{{ vpc_tags }}', - true, - true, - '{{ region }}'; - -/*+ statecheck, retries=5, retry_delay=5 */ -SELECT COUNT(*) as count FROM -( -SELECT vpc_id, -cidr_block, -json_group_object(tag_key, tag_value) as tags -FROM aws.ec2.vpc_tags -WHERE region = '{{ region }}' -AND cidr_block = '{{ vpc_cidr_block }}' -GROUP BY vpc_id -HAVING json_extract(tags, '$.Provisioner') = 'stackql' -AND json_extract(tags, '$.StackName') = '{{ stack_name }}' -AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}' -) t -WHERE cidr_block = '{{ vpc_cidr_block }}'; - -/*+ exports, retries=5, retry_delay=5 */ -SELECT vpc_id, vpc_cidr_block FROM -( -SELECT vpc_id, cidr_block as "vpc_cidr_block", -json_group_object(tag_key, tag_value) as tags -FROM aws.ec2.vpc_tags -WHERE region = '{{ region }}' -AND cidr_block = '{{ vpc_cidr_block }}' -GROUP BY vpc_id -HAVING json_extract(tags, '$.Provisioner') = 'stackql' -AND json_extract(tags, '$.StackName') = '{{ stack_name }}' -AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}' -) t; - -/*+ delete */ -DELETE FROM aws.ec2.vpcs -WHERE data__Identifier = '{{ vpc_id }}' -AND region = '{{ region }}'; \ No newline at end of file diff --git a/examples/confluent/cmd-specific-auth/stackql_manifest.yml b/examples/confluent/cmd-specific-auth/stackql_manifest.yml deleted file mode 100644 index 7450964..0000000 --- a/examples/confluent/cmd-specific-auth/stackql_manifest.yml +++ /dev/null @@ -1,40 +0,0 @@ -# -# aws starter project manifest file, add and update values as needed -# -version: 1 -name: "cmd-specific-auth" -description: description for "cmd-specific-auth" -providers: - - aws -globals: - - name: region - description: aws region - value: "{{ AWS_REGION }}" - - name: global_tags - value: - - Key: Provisioner - Value: stackql - - Key: StackName - Value: "{{ stack_name }}" - - Key: StackEnv - Value: "{{ stack_env }}" -resources: - - name: example_vpc - description: example vpc resource - props: - - name: vpc_cidr_block - values: - prd: - value: "10.0.0.0/16" - sit: - value: "10.1.0.0/16" - dev: - value: "10.2.0.0/16" - - name: vpc_tags - value: - - Key: Name - Value: "{{ stack_name }}-{{ stack_env }}-vpc" - merge: ['global_tags'] - exports: - - vpc_id - - vpc_cidr_block \ No newline at end of file diff --git a/examples/databricks/classic/resources/aws/iam/iam_role.iql b/examples/databricks/classic/resources/aws/iam/iam_role.iql index eebcfcd..d671655 100644 --- a/examples/databricks/classic/resources/aws/iam/iam_role.iql +++ b/examples/databricks/classic/resources/aws/iam/iam_role.iql @@ -1,7 +1,7 @@ /*+ exists */ SELECT COUNT(*) as count FROM aws.iam.roles -WHERE data__Identifier = '{{ role_name }}' +WHERE Identifier = '{{ role_name }}' /*+ create */ INSERT INTO aws.iam.roles ( @@ -24,14 +24,14 @@ SELECT /*+ update */ update aws.iam.roles -set data__PatchDocument = string('{{ { +set PatchDocument = string('{{ { "Description": description, "Path": path, "AssumeRolePolicyDocument": assume_role_policy_document, "Policies": policies, "Tags": global_tags } | generate_patch_document }}') -WHERE data__Identifier = '{{ role_name }}' +WHERE Identifier = '{{ role_name }}' AND region = 'us-east-1'; /*+ statecheck, retries=3, retry_delay=5 */ @@ -42,7 +42,7 @@ SELECT COUNT(*) as count FROM ( JSON_EQUAL(assume_role_policy_document, '{{ assume_role_policy_document }}') as test_assume_role_policy_doc, JSON_EQUAL(policies, '{{ policies }}') as test_policies FROM aws.iam.roles - WHERE data__Identifier = '{{ role_name }}')t + WHERE Identifier = '{{ role_name }}')t WHERE test_assume_role_policy_doc = 1 AND test_policies = 1 AND path = '{{ path }}'; @@ -52,9 +52,9 @@ SELECT '{{ role_name }}' as aws_iam_role_name, arn as aws_iam_role_arn FROM aws.iam.roles -WHERE data__Identifier = '{{ role_name }}' +WHERE Identifier = '{{ role_name }}' /*+ delete */ DELETE FROM aws.iam.roles -WHERE data__Identifier = '{{ role_name }}' +WHERE Identifier = '{{ role_name }}' AND region = 'us-east-1' \ No newline at end of file diff --git a/examples/databricks/classic/resources/aws/s3/workspace_bucket.iql b/examples/databricks/classic/resources/aws/s3/workspace_bucket.iql index a20c908..b4e3fe6 100644 --- a/examples/databricks/classic/resources/aws/s3/workspace_bucket.iql +++ b/examples/databricks/classic/resources/aws/s3/workspace_bucket.iql @@ -2,7 +2,7 @@ SELECT COUNT(*) as count FROM aws.s3.buckets WHERE region = '{{ region }}' -AND data__Identifier = '{{ bucket_name }}' +AND Identifier = '{{ bucket_name }}' /*+ create */ INSERT INTO aws.s3.buckets ( @@ -25,7 +25,7 @@ SELECT /*+ update */ update aws.s3.buckets -set data__PatchDocument = string('{{ { +set PatchDocument = string('{{ { "OwnershipControls": ownership_controls, "BucketEncryption": bucket_encryption, "PublicAccessBlockConfiguration": public_access_block_configuration, @@ -34,7 +34,7 @@ set data__PatchDocument = string('{{ { } | generate_patch_document }}') WHERE region = '{{ region }}' -AND data__Identifier = '{{ bucket_name }}' +AND Identifier = '{{ bucket_name }}' /*+ statecheck, retries=3, retry_delay=5 */ SELECT COUNT(*) as count FROM ( @@ -45,7 +45,7 @@ SELECT COUNT(*) as count FROM ( JSON_EQUAL(versioning_configuration, '{{ versioning_configuration }}') as test_versioning_configuration FROM aws.s3.buckets WHERE region = '{{ region }}' - AND data__Identifier = '{{ bucket_name }}' + AND Identifier = '{{ bucket_name }}' )t WHERE test_ownership_controls = 1 AND test_encryption = 1 @@ -58,4 +58,4 @@ arn as aws_s3_workspace_bucket_arn, bucket_name as aws_s3_workspace_bucket_name FROM aws.s3.buckets WHERE region = '{{ region }}' -AND data__Identifier = '{{ bucket_name }}' +AND Identifier = '{{ bucket_name }}' diff --git a/examples/databricks/classic/resources/aws/s3/workspace_bucket_policy.iql b/examples/databricks/classic/resources/aws/s3/workspace_bucket_policy.iql index cead151..5e6f09d 100644 --- a/examples/databricks/classic/resources/aws/s3/workspace_bucket_policy.iql +++ b/examples/databricks/classic/resources/aws/s3/workspace_bucket_policy.iql @@ -19,12 +19,12 @@ SELECT /*+ update */ update aws.s3.bucket_policies -set data__PatchDocument = string('{{ { +set PatchDocument = string('{{ { "PolicyDocument": policy_document } | generate_patch_document }}') WHERE region = '{{ region }}' -AND data__Identifier = '{{ aws_s3_workspace_bucket_name }}'; +AND Identifier = '{{ aws_s3_workspace_bucket_name }}'; /*+ statecheck, retries=3, retry_delay=5 */ SELECT COUNT(*) as count FROM ( @@ -32,5 +32,5 @@ SELECT COUNT(*) as count FROM ( JSON_EQUAL(policy_document, '{{ policy_document }}') as test_policy_document FROM aws.s3.bucket_policies WHERE region = '{{ region }}' - AND data__Identifier = '{{ aws_s3_workspace_bucket_name }}')t + AND Identifier = '{{ aws_s3_workspace_bucket_name }}')t WHERE test_policy_document = 1; diff --git a/examples/databricks/classic/resources/aws/vpc/elastic_ip.iql b/examples/databricks/classic/resources/aws/vpc/elastic_ip.iql index d4dd982..dca2694 100644 --- a/examples/databricks/classic/resources/aws/vpc/elastic_ip.iql +++ b/examples/databricks/classic/resources/aws/vpc/elastic_ip.iql @@ -52,5 +52,5 @@ AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}' /*+ delete */ DELETE FROM aws.ec2.eips -WHERE data__Identifier = '{{ eip_public_id }}|{{ eip_allocation_id}}' +WHERE Identifier = '{{ eip_public_id }}|{{ eip_allocation_id}}' AND region = '{{ region }}' diff --git a/examples/databricks/classic/resources/aws/vpc/inet_gateway.iql b/examples/databricks/classic/resources/aws/vpc/inet_gateway.iql index dc42032..c7f53cd 100644 --- a/examples/databricks/classic/resources/aws/vpc/inet_gateway.iql +++ b/examples/databricks/classic/resources/aws/vpc/inet_gateway.iql @@ -50,5 +50,5 @@ AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}' /*+ delete */ DELETE FROM aws.ec2.internet_gateways -WHERE data__Identifier = '{{ internet_gateway_id }}' +WHERE Identifier = '{{ internet_gateway_id }}' AND region = '{{ region }}'; diff --git a/examples/databricks/classic/resources/aws/vpc/inet_gw_attachment.iql b/examples/databricks/classic/resources/aws/vpc/inet_gw_attachment.iql index 31b9d25..000c5cd 100644 --- a/examples/databricks/classic/resources/aws/vpc/inet_gw_attachment.iql +++ b/examples/databricks/classic/resources/aws/vpc/inet_gw_attachment.iql @@ -35,5 +35,5 @@ AND vpc_id = '{{ vpc_id }}' /*+ delete */ DELETE FROM aws.ec2.vpc_gateway_attachments -WHERE data__Identifier = 'IGW|{{ vpc_id }}' +WHERE Identifier = 'IGW|{{ vpc_id }}' AND region = '{{ region }}'; \ No newline at end of file diff --git a/examples/databricks/classic/resources/aws/vpc/inet_route.iql b/examples/databricks/classic/resources/aws/vpc/inet_route.iql index b46cc0f..3136204 100644 --- a/examples/databricks/classic/resources/aws/vpc/inet_route.iql +++ b/examples/databricks/classic/resources/aws/vpc/inet_route.iql @@ -1,10 +1,10 @@ /*+ exists */ SELECT COUNT(*) as count FROM ( -SELECT data__Identifier +SELECT Identifier FROM aws.ec2.routes WHERE region = '{{ region }}' -AND data__Identifier = '{{ route_table_id }}|0.0.0.0/0' +AND Identifier = '{{ route_table_id }}|0.0.0.0/0' ) t /*+ create */ @@ -23,19 +23,19 @@ SELECT /*+ statecheck, retries=5, retry_delay=5 */ SELECT COUNT(*) as count FROM ( -SELECT data__Identifier +SELECT Identifier FROM aws.ec2.routes WHERE region = '{{ region }}' -AND data__Identifier = '{{ route_table_id }}|0.0.0.0/0' +AND Identifier = '{{ route_table_id }}|0.0.0.0/0' ) t /*+ exports, retries=3, retry_delay=5 */ -SELECT data__Identifier as inet_route_indentifer +SELECT Identifier as inet_route_indentifer FROM aws.ec2.routes WHERE region = '{{ region }}' -AND data__Identifier = '{{ route_table_id }}|0.0.0.0/0'; +AND Identifier = '{{ route_table_id }}|0.0.0.0/0'; /*+ delete */ DELETE FROM aws.ec2.routes -WHERE data__Identifier = '{{ inet_route_indentifer }}' +WHERE Identifier = '{{ inet_route_indentifer }}' AND region = '{{ region }}'; \ No newline at end of file diff --git a/examples/databricks/classic/resources/aws/vpc/nat_gateway.iql b/examples/databricks/classic/resources/aws/vpc/nat_gateway.iql index 081fbd2..6cf4185 100644 --- a/examples/databricks/classic/resources/aws/vpc/nat_gateway.iql +++ b/examples/databricks/classic/resources/aws/vpc/nat_gateway.iql @@ -49,5 +49,5 @@ AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}' /*+ delete */ DELETE FROM aws.ec2.nat_gateways -WHERE data__Identifier = '{{ nat_gateway_id }}' +WHERE Identifier = '{{ nat_gateway_id }}' AND region = '{{ region }}'; \ No newline at end of file diff --git a/examples/databricks/classic/resources/aws/vpc/nat_inet_route.iql b/examples/databricks/classic/resources/aws/vpc/nat_inet_route.iql index 9e750f6..c00e963 100644 --- a/examples/databricks/classic/resources/aws/vpc/nat_inet_route.iql +++ b/examples/databricks/classic/resources/aws/vpc/nat_inet_route.iql @@ -1,10 +1,10 @@ /*+ exists */ SELECT COUNT(*) as count FROM ( -SELECT data__Identifier +SELECT Identifier FROM aws.ec2.routes WHERE region = '{{ region }}' -AND data__Identifier = '{{ route_table_id }}|0.0.0.0/0' +AND Identifier = '{{ route_table_id }}|0.0.0.0/0' ) t /*+ create */ @@ -23,19 +23,19 @@ SELECT /*+ statecheck, retries=5, retry_delay=5 */ SELECT COUNT(*) as count FROM ( -SELECT data__Identifier +SELECT Identifier FROM aws.ec2.routes WHERE region = '{{ region }}' -AND data__Identifier = '{{ route_table_id }}|0.0.0.0/0' +AND Identifier = '{{ route_table_id }}|0.0.0.0/0' ) t /*+ exports, retries=3, retry_delay=5 */ -SELECT data__Identifier as nat_inet_route_indentifer +SELECT Identifier as nat_inet_route_indentifer FROM aws.ec2.routes WHERE region = '{{ region }}' -AND data__Identifier = '{{ route_table_id }}|0.0.0.0/0'; +AND Identifier = '{{ route_table_id }}|0.0.0.0/0'; /*+ delete */ DELETE FROM aws.ec2.routes -WHERE data__Identifier = '{{ inet_route_indentifer }}' +WHERE Identifier = '{{ inet_route_indentifer }}' AND region = '{{ region }}'; \ No newline at end of file diff --git a/examples/databricks/classic/resources/aws/vpc/route_table.iql b/examples/databricks/classic/resources/aws/vpc/route_table.iql index 7b0aa76..de399f9 100644 --- a/examples/databricks/classic/resources/aws/vpc/route_table.iql +++ b/examples/databricks/classic/resources/aws/vpc/route_table.iql @@ -50,5 +50,5 @@ AND json_extract(tags, '$.Name') = '{{ route_table_name }}' /*+ delete */ DELETE FROM aws.ec2.route_tables -WHERE data__Identifier = '{{ route_table_id }}' +WHERE Identifier = '{{ route_table_id }}' AND region = '{{ region }}'; \ No newline at end of file diff --git a/examples/databricks/classic/resources/aws/vpc/security_group.iql b/examples/databricks/classic/resources/aws/vpc/security_group.iql index 15e9061..d52fec8 100644 --- a/examples/databricks/classic/resources/aws/vpc/security_group.iql +++ b/examples/databricks/classic/resources/aws/vpc/security_group.iql @@ -37,5 +37,5 @@ AND group_name = '{{ group_name }}' /*+ delete */ DELETE FROM aws.ec2.security_groups -WHERE data__Identifier = '{{ security_group_id }}' +WHERE Identifier = '{{ security_group_id }}' AND region = '{{ region }}'; \ No newline at end of file diff --git a/examples/databricks/classic/resources/aws/vpc/security_group_rules.iql b/examples/databricks/classic/resources/aws/vpc/security_group_rules.iql index 62f79eb..4536912 100644 --- a/examples/databricks/classic/resources/aws/vpc/security_group_rules.iql +++ b/examples/databricks/classic/resources/aws/vpc/security_group_rules.iql @@ -2,16 +2,16 @@ SELECT COUNT(*) as count FROM aws.ec2.security_groups WHERE region = '{{ region }}' -AND data__Identifier = '{{ security_group_id }}' +AND Identifier = '{{ security_group_id }}' /*+ createorupdate */ update aws.ec2.security_groups -set data__PatchDocument = string('{{ { +set PatchDocument = string('{{ { "SecurityGroupIngress": security_group_ingress, "SecurityGroupEgress": security_group_egress } | generate_patch_document }}') WHERE region = '{{ region }}' -AND data__Identifier = '{{ security_group_id }}' +AND Identifier = '{{ security_group_id }}' /*+ statecheck, retries=5, retry_delay=5 */ SELECT COUNT(*) as count FROM @@ -21,7 +21,7 @@ SELECT COUNT(*) as count FROM JSON_EQUAL(security_group_egress, '{{ security_group_egress }}') as egress_test FROM aws.ec2.security_groups WHERE region = '{{ region }}' - AND data__Identifier = '{{ security_group_id }}' + AND Identifier = '{{ security_group_id }}' AND ingress_test = 1 AND egress_test = 1 ) t; diff --git a/examples/databricks/classic/resources/aws/vpc/subnet.iql b/examples/databricks/classic/resources/aws/vpc/subnet.iql index 83667f5..629c638 100644 --- a/examples/databricks/classic/resources/aws/vpc/subnet.iql +++ b/examples/databricks/classic/resources/aws/vpc/subnet.iql @@ -39,5 +39,5 @@ AND cidr_block = '{{ cidr_block }}'; /*+ delete */ DELETE FROM aws.ec2.subnets -WHERE data__Identifier = '{{ subnet_id }}' +WHERE Identifier = '{{ subnet_id }}' AND region = '{{ region }}'; \ No newline at end of file diff --git a/examples/databricks/classic/resources/aws/vpc/subnet_rt_assn.iql b/examples/databricks/classic/resources/aws/vpc/subnet_rt_assn.iql index d0c8b33..ec3762d 100644 --- a/examples/databricks/classic/resources/aws/vpc/subnet_rt_assn.iql +++ b/examples/databricks/classic/resources/aws/vpc/subnet_rt_assn.iql @@ -30,5 +30,5 @@ AND subnet_id = '{{ subnet_id }}'; /*+ delete */ DELETE FROM aws.ec2.subnet_route_table_associations -WHERE data__Identifier = '{{ route_table_assn_id }}' +WHERE Identifier = '{{ route_table_assn_id }}' AND region = '{{ region }}'; \ No newline at end of file diff --git a/examples/databricks/classic/resources/aws/vpc/tag_main_vpc_route_table.iql b/examples/databricks/classic/resources/aws/vpc/tag_main_vpc_route_table.iql index cc03c2a..d438f9a 100644 --- a/examples/databricks/classic/resources/aws/vpc/tag_main_vpc_route_table.iql +++ b/examples/databricks/classic/resources/aws/vpc/tag_main_vpc_route_table.iql @@ -1,7 +1,7 @@ /*+ command */ update aws.ec2.route_tables -set data__PatchDocument = string('{{ { +set PatchDocument = string('{{ { "Tags": tags } | generate_patch_document }}') WHERE region = '{{ region }}' -AND data__Identifier = '{{ main_route_table_id }}'; \ No newline at end of file +AND Identifier = '{{ main_route_table_id }}'; \ No newline at end of file diff --git a/examples/databricks/classic/resources/aws/vpc/vpc.iql b/examples/databricks/classic/resources/aws/vpc/vpc.iql index 56e1c54..de01fa5 100644 --- a/examples/databricks/classic/resources/aws/vpc/vpc.iql +++ b/examples/databricks/classic/resources/aws/vpc/vpc.iql @@ -56,5 +56,5 @@ AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}' /*+ delete */ DELETE FROM aws.ec2.vpcs -WHERE data__Identifier = '{{ vpc_id}}' +WHERE Identifier = '{{ vpc_id}}' AND region = '{{ region }}'; \ No newline at end of file diff --git a/examples/databricks/classic/resources/aws/vpc/vpc_endpoint.iql b/examples/databricks/classic/resources/aws/vpc/vpc_endpoint.iql index d40f522..e9f6591 100644 --- a/examples/databricks/classic/resources/aws/vpc/vpc_endpoint.iql +++ b/examples/databricks/classic/resources/aws/vpc/vpc_endpoint.iql @@ -56,5 +56,5 @@ AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}'; /*+ delete */ DELETE FROM aws.ec2.vpc_endpoints -WHERE data__Identifier = 's3_gateway_endpoint_id' +WHERE Identifier = 's3_gateway_endpoint_id' AND region = 'us-east-1'; \ No newline at end of file diff --git a/examples/databricks/classic/resources/databricks_account/credentials.iql b/examples/databricks/classic/resources/databricks_account/credentials.iql index c0d8327..9221933 100644 --- a/examples/databricks/classic/resources/databricks_account/credentials.iql +++ b/examples/databricks/classic/resources/databricks_account/credentials.iql @@ -7,8 +7,8 @@ AND credentials_name = '{{ credentials_name }}' /*+ create */ INSERT INTO databricks_account.provisioning.credentials ( account_id, -data__credentials_name, -data__aws_credentials +credentials_name, +aws_credentials ) SELECT '{{ databricks_account_id }}', diff --git a/examples/databricks/classic/resources/databricks_account/network.iql b/examples/databricks/classic/resources/databricks_account/network.iql index 45e0b0a..98623b2 100644 --- a/examples/databricks/classic/resources/databricks_account/network.iql +++ b/examples/databricks/classic/resources/databricks_account/network.iql @@ -7,10 +7,10 @@ AND network_name = '{{ databricks_network_name }}' /*+ create */ INSERT INTO databricks_account.provisioning.networks ( account_id, -data__network_name, -data__vpc_id, -data__subnet_ids, -data__security_group_ids +network_name, +vpc_id, +subnet_ids, +security_group_ids ) SELECT '{{ databricks_account_id }}', diff --git a/examples/databricks/classic/resources/databricks_account/storage_configuration.iql b/examples/databricks/classic/resources/databricks_account/storage_configuration.iql index 4e60cfc..baa5a67 100644 --- a/examples/databricks/classic/resources/databricks_account/storage_configuration.iql +++ b/examples/databricks/classic/resources/databricks_account/storage_configuration.iql @@ -7,8 +7,8 @@ AND storage_configuration_name = '{{ storage_configuration_name }}' /*+ create */ INSERT INTO databricks_account.provisioning.storage ( account_id, -data__storage_configuration_name, -data__root_bucket_info +storage_configuration_name, +root_bucket_info ) SELECT '{{ databricks_account_id }}', diff --git a/examples/databricks/classic/resources/databricks_account/update_group_membership.iql b/examples/databricks/classic/resources/databricks_account/update_group_membership.iql index 375d926..7f28c52 100644 --- a/examples/databricks/classic/resources/databricks_account/update_group_membership.iql +++ b/examples/databricks/classic/resources/databricks_account/update_group_membership.iql @@ -1,6 +1,6 @@ /*+ command */ update databricks_account.iam.groups -set data__schemas = '["urn:ietf:params:scim:api:messages:2.0:PatchOp"]', -data__Operations = '[{"op": "replace", "path": "members", "value": {{ databricks_workspace_group_members }} }]' +set schemas = '["urn:ietf:params:scim:api:messages:2.0:PatchOp"]', +Operations = '[{"op": "replace", "path": "members", "value": {{ databricks_workspace_group_members }} }]' WHERE account_id = '{{ databricks_account_id }}' AND id = '{{ databricks_group_id }}'; diff --git a/examples/databricks/classic/resources/databricks_account/workspace.iql b/examples/databricks/classic/resources/databricks_account/workspace.iql index 9da2dea..540aa0e 100644 --- a/examples/databricks/classic/resources/databricks_account/workspace.iql +++ b/examples/databricks/classic/resources/databricks_account/workspace.iql @@ -7,11 +7,11 @@ AND workspace_name = '{{ workspace_name }}' /*+ create */ INSERT INTO databricks_account.provisioning.workspaces ( account_id, -data__workspace_name, -data__aws_region, -data__credentials_id, -data__storage_configuration_id, -data__pricing_tier +workspace_name, +aws_region, +credentials_id, +storage_configuration_id, +pricing_tier ) SELECT '{{ databricks_account_id }}', diff --git a/examples/databricks/classic/resources/databricks_account/workspace_group.iql b/examples/databricks/classic/resources/databricks_account/workspace_group.iql index 4d3494a..616ae13 100644 --- a/examples/databricks/classic/resources/databricks_account/workspace_group.iql +++ b/examples/databricks/classic/resources/databricks_account/workspace_group.iql @@ -7,7 +7,7 @@ AND displayName = '{{ display_name }}' /*+ create */ INSERT INTO databricks_account.iam.groups ( account_id, -data__displayName +displayName ) SELECT '{{ databricks_account_id }}', diff --git a/examples/databricks/classic/resources/databricks_account/workspace_permission_assignments.iql b/examples/databricks/classic/resources/databricks_account/workspace_permission_assignments.iql index 00387e3..6051aba 100644 --- a/examples/databricks/classic/resources/databricks_account/workspace_permission_assignments.iql +++ b/examples/databricks/classic/resources/databricks_account/workspace_permission_assignments.iql @@ -10,7 +10,7 @@ INSERT INTO databricks_account.iam.workspace_permission_assignments ( account_id, principal_id, workspace_id, -data__permissions +permissions ) SELECT '{{ databricks_account_id }}', diff --git a/examples/databricks/classic/resources/databricks_workspace/all_purpose_cluster.iql b/examples/databricks/classic/resources/databricks_workspace/all_purpose_cluster.iql index 44b3703..18e84bf 100644 --- a/examples/databricks/classic/resources/databricks_workspace/all_purpose_cluster.iql +++ b/examples/databricks/classic/resources/databricks_workspace/all_purpose_cluster.iql @@ -7,17 +7,17 @@ AND cluster_name = '{{ cluster_name }}' /*+ create */ INSERT INTO databricks_workspace.compute.clusters ( deployment_name, -data__cluster_name, -data__num_workers, -data__is_single_node, -data__kind, -data__spark_version, -data__node_type_id, -data__data_security_mode, -data__runtime_engine, -data__single_user_name, -data__aws_attributes, -data__custom_tags +cluster_name, +num_workers, +is_single_node, +kind, +spark_version, +node_type_id, +data_security_mode, +runtime_engine, +single_user_name, +aws_attributes, +custom_tags ) SELECT '{{ databricks_deployment_name }}', diff --git a/examples/databricks/serverless-old/README.md b/examples/databricks/serverless-old/README.md deleted file mode 100644 index 409894d..0000000 --- a/examples/databricks/serverless-old/README.md +++ /dev/null @@ -1,245 +0,0 @@ -# `stackql-deploy` example project for `databricks` - -This exercise is to bootstrap a databricks / aws tenancy using `stackql-deploy`. It is an important use case for platform bootstrap and we are excited to perform it with the `stackql` toolchain. We hope you enjoy and find this valuable. Please drop us a note with your forthright opinion on this and check out our issues on github. - -## A word of caution - -Please take the greatest care in performing this exercise; it will incur expenses, as it involves creating (and destroying) resources which cost money. Please be aware that you **must** cancel your databricks subscription after completing this exercise, otherwise you will incur ongoing expenses. That is, do **not** skip the section [Cancel databricks subscription](#cancel-databricks-subsription). We strongly advise that you verify all resources are destroyed at the conclusion of this exercise. Web pages and certain behaviours may change, so please be thorough in your verification. We will keep this page up-to-date on a best effort basis only. It is very much a case of owner onus applies. - -## Manual Setup - -Dependencies: - -- aws Account Created. -- Required clickops to set up databricks on aws: - - Turn on aws Marketplace `databricks` offering, using [the aws manage subscriptions page](https://console.aws.amazon.com/marketplace/home#/subscriptions), per Figure S1. - - Follow the suggested setup flow as directed, from this page. These clickops steps are necessary at this time for initial account setup. The way I followed this, it created a workspace for me at setup, per Figure S3. We shall not use this one and rather, later on we shall dispose of it; because we do not trust auto-created resources out of hand. In the process of creating the databricks subscription, a second aws account is created. - - Copy the databricks account id from basically any web page in the databricks console. This is done by clicking on the user icon at the top RHS and then the UI provides a copy shortcut, per Fugure U1. Save this locally for later use, expanded below. - - We need the aws account id that was created for the databricks subscription. It is not exactly heralded by the web pages, nor is it actively hidden. It can be captured in a couple of places, including the databricks storage account creatted in the subscription flow, per Figure XA1. copy and save this locally for later use, expanded below. - - Create a service principal to use as a "CICD agent", using the page shown in Figure S4. - - Grant the CICD agent account admin role, using the page shown in Figure S5. - - Create a secret for the CICD agent, using the page shown in Figure S6. At the time you create this, you will need to safely store the client secret and client id, as prompted by the web page. These will be used below. -- Setup your virtual environment, from the root of this repository `cicd/setup/setup-env.sh`. - -Now, is is convenient to use environment variables for context. Note that for our example, there is only one aws account apropos, however this is not always the case for an active professional, so while `DATABRICKS_AWS_ACCOUNT_ID` is the same as `AWS_ACCOUNT_ID` here, it need not always be the case. Create a file in the path `examples/databricks/serverless/sec/env.sh` (relative to the root of this repository) with contents of the form: - -```bash -#!/usr/bin/env bash - -export AWS_REGION='us-east-1' # or wherever you want -export AWS_ACCOUNT_ID='' -export DATABRICKS_ACCOUNT_ID='' -export DATABRICKS_AWS_ACCOUNT_ID='' - -# These need to be created by clickops under [the account level user managment page](https://accounts.cloud.databricks.com/user-management). -export DATABRICKS_CLIENT_ID='' -export DATABRICKS_CLIENT_SECRET='' - -## These can be skipped if you run on [aws cloud shell](https://docs.aws.amazon.com/cloudshell/latest/userguide/welcome.html). -export AWS_SECRET_ACCESS_KEY='' -export AWS_ACCESS_KEY_ID='' - -``` - -## Optional step: sanity checks with stackql - -Now, let us do some sanity checks and housekeeping with `stackql`. This is purely optional. From the root of this repository: - -``` -source examples/databricks/serverless/convenience.sh -stackql shell -``` - -This will start a `stackql` interactive shell. Here are some commands you can run (I will not place output here, that will be shared in a corresponding video): - - -```sql -registry pull databricks_account v24.12.00279; -registry pull databricks_workspace v24.12.00279; - --- This will fail if accounts, subscription, or credentials are in error. -select account_id FROM databricks_account.provisioning.credentials WHERE account_id = ''; -select account_id, workspace_name, workspace_id, workspace_status from databricks_account.provisioning.workspaces where account_id = ''; -``` - -For extra credit, you can (asynchronously) delete the unnecessary workspace with `delete from databricks_account.provisioning.workspaces where account_id = '' and workspace_id = '';`, where you obtain the workspace id from the above query. I have noted that due to some reponse caching it takes a while to disappear from select queries (much longer than disappearance from the web page), and you may want to bounce the `stackql` session to hurry things along. This is not happening on the `stackql` side, but session bouncing forces a token refresh which can help cache busting. - -## Lifecycle management - -Time to get down to business. From the root of this repository: - -```bash -python3 -m venv myenv -source examples/databricks/serverless/convenience.sh -source venv/bin/activate -pip install stackql-deploy -``` - -> alternatively set the `AWS_REGION`, `AWS_ACCOUNT_ID`, `DATABRICKS_ACCOUNT_ID`, `DATABRICKS_AWS_ACCOUNT_ID` along with provider credentials `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `DATABRICKS_CLIENT_ID`, `DATABRICKS_CLIENT_SECRET` - -Then, do a dry run (good for catching **some** environmental issues): - -```bash -stackql-deploy build \ -examples/databricks/serverless dev \ --e AWS_REGION=${AWS_REGION} \ --e AWS_ACCOUNT_ID=${AWS_ACCOUNT_ID} \ --e DATABRICKS_ACCOUNT_ID=${DATABRICKS_ACCOUNT_ID} \ --e DATABRICKS_AWS_ACCOUNT_ID=${DATABRICKS_AWS_ACCOUNT_ID} \ ---dry-run -``` - -You will see a verbose rendition of what `stackql-deploy` intends to do. - - -Now, let use do it for real: - -```bash -stackql-deploy build \ -examples/databricks/serverless dev \ --e AWS_REGION=${AWS_REGION} \ --e AWS_ACCOUNT_ID=${AWS_ACCOUNT_ID} \ --e DATABRICKS_ACCOUNT_ID=${DATABRICKS_ACCOUNT_ID} \ --e DATABRICKS_AWS_ACCOUNT_ID=${DATABRICKS_AWS_ACCOUNT_ID} \ ---show-queries -``` - -The output is quite verbose, concludes in: - -``` -2025-02-08 12:51:25,914 - stackql-deploy - INFO - 📤 set [databricks_workspace_id] to [482604062392118] in exports -2025-02-08 12:51:25,915 - stackql-deploy - INFO - ✅ successfully deployed databricks_workspace -2025-02-08 12:51:25,915 - stackql-deploy - INFO - deployment completed in 0:04:09.603631 -🚀 build complete -``` - -Success!!! - -We can also use `stackql-deploy` to assess if our infra is shipshape: - -```bash -stackql-deploy test \ -examples/databricks/serverless dev \ --e AWS_REGION=${AWS_REGION} \ --e AWS_ACCOUNT_ID=${AWS_ACCOUNT_ID} \ --e DATABRICKS_ACCOUNT_ID=${DATABRICKS_ACCOUNT_ID} \ --e DATABRICKS_AWS_ACCOUNT_ID=${DATABRICKS_AWS_ACCOUNT_ID} \ ---show-queries -``` - -Again, the output is quite verbose, concludes in: - -``` -2025-02-08 13:15:45,821 - stackql-deploy - INFO - 📤 set [databricks_workspace_id] to [482604062392118] in exports -2025-02-08 13:15:45,821 - stackql-deploy - INFO - ✅ test passed for databricks_workspace -2025-02-08 13:15:45,821 - stackql-deploy - INFO - deployment completed in 0:02:30.255860 -🔍 tests complete (dry run: False) -``` - -Success!!! - -Now, let us teardown our `stackql-deploy` managed infra: - -```bash -stackql-deploy teardown \ -examples/databricks/serverless dev \ --e AWS_REGION=${AWS_REGION} \ --e AWS_ACCOUNT_ID=${AWS_ACCOUNT_ID} \ --e DATABRICKS_ACCOUNT_ID=${DATABRICKS_ACCOUNT_ID} \ --e DATABRICKS_AWS_ACCOUNT_ID=${DATABRICKS_AWS_ACCOUNT_ID} \ ---show-queries -``` - -Takes its time, again verbose, concludes in: - -``` -2025-02-08 13:24:17,941 - stackql-deploy - INFO - ✅ successfully deleted AWS_iam_cross_account_role -2025-02-08 13:24:17,942 - stackql-deploy - INFO - deployment completed in 0:03:21.191788 -🚧 teardown complete (dry run: False) -``` - -Success!!! - -## Optional step: verify destruction with stackql - -Now, let us do some sanity checks and housekeeping with `stackql`. This is purely optional. From the root of this repository: - -``` - -source examples/databricks/serverless/convenience.sh - -stackql shell - -``` - -This will start a `stackql` interactive shell. Here are some commands you can run (I will not place output here): - - -```sql - -registry pull databricks_account v24.12.00279; - -registry pull databricks_workspace v24.12.00279; - - - -select account_id, workspace_name, workspace_id, workspace_status from databricks_account.provisioning.workspaces where account_id = ''; - -``` - -## Cancel databricks subsription - -This is **very** important. - -Go to [the aws Marketplace manage subscriptions page](https://console.aws.amazon.com/marketplace/home#/subscriptions), navigate to databricks and then cancel the subscription. - -## Figures - - -![Create aws databricks subscription](/examples/databricks/serverless/assets/create-aws-databricks-subscription.png) - -**Figure S1**: Create aws databricks subscription. - ---- - -![Awaiting aws databricks subscription resources](/examples/databricks/serverless/assets/awaiting-subscription-resources.png) - -**Figure S2**: Awaiting aws databricks subscription resources. - ---- - -![Auto provisioned workspace](/examples/databricks/serverless/assets/auto-provisioned-worskpace.png) - -**Figure S3**: Auto provisioned workspace. - ---- - -![Capture databricks account id](/examples/databricks/serverless/assets/capture-databricks-account-id.png) - -**Figure U1**: Capture databricks account id. - ---- - -![Capture cross databricks aws account id](/examples/databricks/serverless/assets/capture-cross-databricks-aws-account-id.png) - -**Figure XA1**: Capture cross databricks aws account id. - ---- - -![Create CICD agent](/examples/databricks/serverless/assets/create-cicd-agent.png) - -**Figure S4**: Create CICD agent. - ---- - -![Grant account admin to CICD agent](/examples/databricks/serverless/assets/grant-account-admin-cicd-agent.png) - -**Figure S5**: Grant account admin to CICD agent. - ---- - -![Generate secret for CICD agent](/examples/databricks/serverless/assets/generate-secret-ui.png) - -**Figure S6**: Generate secret for CICD agent. - ---- diff --git a/examples/databricks/serverless-old/assets/auto-provisioned-worskpace.png b/examples/databricks/serverless-old/assets/auto-provisioned-worskpace.png deleted file mode 100644 index a9fbcb6..0000000 Binary files a/examples/databricks/serverless-old/assets/auto-provisioned-worskpace.png and /dev/null differ diff --git a/examples/databricks/serverless-old/assets/awaiting-subscription-resources.png b/examples/databricks/serverless-old/assets/awaiting-subscription-resources.png deleted file mode 100644 index 9505100..0000000 Binary files a/examples/databricks/serverless-old/assets/awaiting-subscription-resources.png and /dev/null differ diff --git a/examples/databricks/serverless-old/assets/capture-cross-databricks-aws-account-id.png b/examples/databricks/serverless-old/assets/capture-cross-databricks-aws-account-id.png deleted file mode 100644 index 6fdb3c4..0000000 Binary files a/examples/databricks/serverless-old/assets/capture-cross-databricks-aws-account-id.png and /dev/null differ diff --git a/examples/databricks/serverless-old/assets/capture-databricks-account-id.png b/examples/databricks/serverless-old/assets/capture-databricks-account-id.png deleted file mode 100644 index c890299..0000000 Binary files a/examples/databricks/serverless-old/assets/capture-databricks-account-id.png and /dev/null differ diff --git a/examples/databricks/serverless-old/assets/create-aws-databricks-subscription.png b/examples/databricks/serverless-old/assets/create-aws-databricks-subscription.png deleted file mode 100644 index b5c9e7f..0000000 Binary files a/examples/databricks/serverless-old/assets/create-aws-databricks-subscription.png and /dev/null differ diff --git a/examples/databricks/serverless-old/assets/create-cicd-agent.png b/examples/databricks/serverless-old/assets/create-cicd-agent.png deleted file mode 100644 index faf1643..0000000 Binary files a/examples/databricks/serverless-old/assets/create-cicd-agent.png and /dev/null differ diff --git a/examples/databricks/serverless-old/assets/generate-secret-ui.png b/examples/databricks/serverless-old/assets/generate-secret-ui.png deleted file mode 100644 index daf4f23..0000000 Binary files a/examples/databricks/serverless-old/assets/generate-secret-ui.png and /dev/null differ diff --git a/examples/databricks/serverless-old/assets/grant-account-admin-cicd-agent.png b/examples/databricks/serverless-old/assets/grant-account-admin-cicd-agent.png deleted file mode 100644 index f50e0c0..0000000 Binary files a/examples/databricks/serverless-old/assets/grant-account-admin-cicd-agent.png and /dev/null differ diff --git a/examples/databricks/serverless-old/convenience.sh b/examples/databricks/serverless-old/convenience.sh deleted file mode 100644 index 290ef42..0000000 --- a/examples/databricks/serverless-old/convenience.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash - -CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -export REPOSITORY_ROOT="$(realpath $CURRENT_DIR/../../..)" - - -if [ -f "${REPOSITORY_ROOT}/examples/databricks/serverless/sec/env.sh" ]; -then - source "${REPOSITORY_ROOT}/examples/databricks/serverless/sec/env.sh" -fi - -if [ "${AWS_REGION}" = "" ]; -then - AWS_REGION='us-east-1' -fi - -if [ "${AWS_ACCOUNT_ID}" = "" ]; -then - echo "AWS_ACCOUNT_ID must be set" >&2 - exit 1s -fi - -if [ "${DATABRICKS_ACCOUNT_ID}" = "" ]; -then - echo "DATABRICKS_ACCOUNT_ID must be set" >&2 - exit 1 -fi - -if [ "${DATABRICKS_AWS_ACCOUNT_ID}" = "" ]; -then - echo "DATABRICKS_AWS_ACCOUNT_ID must be set" >&2 - exit 1 -fi - -if [ "${DATABRICKS_CLIENT_ID}" = "" ]; -then - echo "DATABRICKS_CLIENT_ID must be set" >&2 - exit 1 -fi - -if [ "${DATABRICKS_CLIENT_SECRET}" = "" ]; -then - echo "DATABRICKS_CLIENT_SECRET must be set" >&2 - exit 1 -fi - -if [ "${AWS_SECRET_ACCESS_KEY}" = "" ]; -then - echo "AWS_SECRET_ACCESS_KEY must be set" >&2 - exit 1 -fi - -if [ "${AWS_ACCESS_KEY_ID}" = "" ]; -then - echo "AWS_ACCESS_KEY_ID must be set" >&2 - exit 1 -fi - -export AWS_REGION -export AWS_ACCOUNT_ID -export DATABRICKS_ACCOUNT_ID -export DATABRICKS_AWS_ACCOUNT_ID - -export DATABRICKS_CLIENT_ID -export DATABRICKS_CLIENT_SECRET - - -export AWS_SECRET_ACCESS_KEY -export AWS_ACCESS_KEY_ID - - diff --git a/examples/databricks/serverless-old/outputs/deployment.json b/examples/databricks/serverless-old/outputs/deployment.json deleted file mode 100644 index 2e6250c..0000000 --- a/examples/databricks/serverless-old/outputs/deployment.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "stack_name": "stackql-serverless", - "stack_env": "prd", - "databricks_workspace_name": "stackql-serverless-prd-workspace", - "databricks_workspace_id": "4014389171618363", - "databricks_deployment_name": "dbc-5a3a87f7-6914", - "databricks_workspace_status": "RUNNING", - "databricks_workspace_url": "https://dbc-5a3a87f7-6914.cloud.databricks.com", - "elapsed_time": "0:00:31.470746" -} \ No newline at end of file diff --git a/examples/databricks/serverless-old/outputs/test.json b/examples/databricks/serverless-old/outputs/test.json deleted file mode 100644 index 0bb1932..0000000 --- a/examples/databricks/serverless-old/outputs/test.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "stack_name": "stackql-serverless", - "stack_env": "prd", - "databricks_workspace_name": "stackql-serverless-prd-workspace", - "databricks_workspace_id": "4014389171618363", - "databricks_deployment_name": "dbc-5a3a87f7-6914", - "databricks_workspace_status": "RUNNING", - "databricks_workspace_url": "https://dbc-5a3a87f7-6914.cloud.databricks.com", - "elapsed_time": "0:00:18.247444" -} \ No newline at end of file diff --git a/examples/databricks/serverless-old/resources/aws/iam/iam_role.iql b/examples/databricks/serverless-old/resources/aws/iam/iam_role.iql deleted file mode 100644 index 0e7d3a0..0000000 --- a/examples/databricks/serverless-old/resources/aws/iam/iam_role.iql +++ /dev/null @@ -1,69 +0,0 @@ -/*+ exists */ -SELECT COUNT(*) as count -FROM awscc.iam.roles -WHERE Identifier = '{{ RoleName }}'; -- snake case in {{ }} - -/*+ create */ -INSERT INTO awscc.iam.roles ( -AssumeRolePolicyDocument, -Description, -ManagedPolicyArns, -MaxSessionDuration, -Path, -PermissionsBoundary, -Policies, -RoleName, -Tags, -region -) -SELECT -'{{ AssumeRolePolicyDocument }}', -- snake case values -'{{ Description }}', -'{{ ManagedPolicyArns }}', -'{{ MaxSessionDuration }}', -'{{ Path }}', -'{{ PermissionsBoundary }}', -'{{ Policies }}', -'{{ RoleName }}', -'{{ Tags }}', -'us-east-1'; --always us-east-1 for iam - -/*+ update */ -UPDATE awscc.iam.roles -SET PatchDocument = string('{{ { -"AssumeRolePolicyDocument": assume_role_policy_document, -"Description": description, -"ManagedPolicyArns": managed_policy_arns, -"MaxSessionDuration": max_session_duration, -"PermissionsBoundary": permissions_boundary, -"Policies": policies, -"Tags": tags -} | generate_patch_document }}') -WHERE region = 'us-east-1' --always us-east-1 for iam -AND Identifier = '{{ RoleName }}'; -- snake case in {{ }} - -/*+ statecheck, retries=5, retry_delay=5 */ -SELECT COUNT(*) as count FROM ( - SELECT - max_session_duration, - path, - AWS_POLICY_EQUAL(assume_role_policy_document, '{{ assume_role_policy_document }}') as test_assume_role_policy_doc, - AWS_POLICY_EQUAL(policies, '{{ policies }}') as test_policies - FROM aws.iam.roles - WHERE data__Identifier = '{{ role_name }}')t -WHERE test_assume_role_policy_doc = 1 -AND test_policies = 1 -AND path = '{{ path }}'; - -/*+ exports, retries=3, retry_delay=5 */ -SELECT -'{{ role_name }}' as aws_iam_role_name, -arn as aws_iam_role_arn -FROM aws.iam.roles -WHERE data__Identifier = '{{ role_name }}' - -/*+ delete */ -DELETE FROM awscc.iam.roles -WHERE Identifier = '' -- snake case in {{ }} -AND region = 'us-east-1'; - diff --git a/examples/databricks/serverless-old/resources/aws/iam/update_metastore_access_role.iql b/examples/databricks/serverless-old/resources/aws/iam/update_metastore_access_role.iql deleted file mode 100644 index 2339232..0000000 --- a/examples/databricks/serverless-old/resources/aws/iam/update_metastore_access_role.iql +++ /dev/null @@ -1,7 +0,0 @@ -/*+ command */ -update aws.iam.roles -set data__PatchDocument = string('{{ { - "AssumeRolePolicyDocument": assume_role_policy_document - } | generate_patch_document }}') -WHERE data__Identifier = '{{ role_name }}' -AND region = 'us-east-1'; \ No newline at end of file diff --git a/examples/databricks/serverless-old/resources/aws/s3/s3_bucket.iql b/examples/databricks/serverless-old/resources/aws/s3/s3_bucket.iql deleted file mode 100644 index 42741fd..0000000 --- a/examples/databricks/serverless-old/resources/aws/s3/s3_bucket.iql +++ /dev/null @@ -1,58 +0,0 @@ -/*+ exists */ -SELECT COUNT(*) as count -FROM aws.s3.buckets -WHERE region = '{{ region }}' -AND data__Identifier = '{{ bucket_name }}' - -/*+ create */ -INSERT INTO aws.s3.buckets ( - BucketName, - OwnershipControls, - BucketEncryption, - PublicAccessBlockConfiguration, - VersioningConfiguration, - Tags, - region -) -SELECT - '{{ bucket_name }}', - '{{ ownership_controls }}', - '{{ bucket_encryption }}', - '{{ public_access_block_configuration }}', - '{{ versioning_configuration }}', - '{{ global_tags }}', - '{{ region }}' - -/*+ update */ -update aws.s3.buckets -set data__PatchDocument = string('{{ { - "OwnershipControls": ownership_controls, - "BucketEncryption": bucket_encryption, - "PublicAccessBlockConfiguration": public_access_block_configuration, - "VersioningConfiguration": versioning_configuration, - "Tags": global_tags - } | generate_patch_document }}') -WHERE -region = '{{ region }}' -AND data__Identifier = '{{ bucket_name }}' - -/*+ exports, retries=3, retry_delay=5 */ -SELECT -arn, -bucket_name -FROM ( - SELECT - arn, - bucket_name, - JSON_EQUAL(ownership_controls, '{{ ownership_controls }}') as test_ownership_controls, - JSON_EQUAL(bucket_encryption, '{{ bucket_encryption }}') as test_encryption, - JSON_EQUAL(public_access_block_configuration, '{{ public_access_block_configuration }}') as test_public_access_block_configuration, - JSON_EQUAL(versioning_configuration, '{{ versioning_configuration }}') as test_versioning_configuration - FROM aws.s3.buckets - WHERE region = '{{ region }}' - AND data__Identifier = '{{ bucket_name }}' -)t -WHERE test_ownership_controls = 1 -AND test_encryption = 1 -AND test_public_access_block_configuration = 1 -AND test_versioning_configuration = 1 diff --git a/examples/databricks/serverless-old/resources/aws/s3/s3_bucket_policy.iql b/examples/databricks/serverless-old/resources/aws/s3/s3_bucket_policy.iql deleted file mode 100644 index 69ec36b..0000000 --- a/examples/databricks/serverless-old/resources/aws/s3/s3_bucket_policy.iql +++ /dev/null @@ -1,36 +0,0 @@ -/*+ exists */ -SELECT COUNT(*) as count -FROM aws.s3.bucket_policies -WHERE region = '{{ region }}' -AND bucket = '{{ aws_s3_workspace_bucket_name }}'; - -/*+ create */ -INSERT INTO aws.s3.bucket_policies ( - Bucket, - PolicyDocument, - ClientToken, - region -) -SELECT - '{{ aws_s3_workspace_bucket_name }}', - '{{ policy_document }}', - '{{ uuid() }}', - '{{ region }}' - -/*+ update */ -update aws.s3.bucket_policies -set data__PatchDocument = string('{{ { - "PolicyDocument": policy_document - } | generate_patch_document }}') -WHERE -region = '{{ region }}' -AND data__Identifier = '{{ aws_s3_workspace_bucket_name }}'; - -/*+ statecheck, retries=3, retry_delay=5 */ -SELECT COUNT(*) as count FROM ( - SELECT - AWS_POLICY_EQUAL(policy_document, '{{ policy_document }}') as test_policy_document - FROM aws.s3.bucket_policies - WHERE region = '{{ region }}' - AND data__Identifier = '{{ aws_s3_workspace_bucket_name }}')t -WHERE test_policy_document = 1; diff --git a/examples/databricks/serverless-old/resources/databricks_account/credentials.iql b/examples/databricks/serverless-old/resources/databricks_account/credentials.iql deleted file mode 100644 index 687b3f1..0000000 --- a/examples/databricks/serverless-old/resources/databricks_account/credentials.iql +++ /dev/null @@ -1,31 +0,0 @@ -/*+ exists */ -SELECT COUNT(*) as count -FROM databricks_account.provisioning.credentials -WHERE account_id = '{{ databricks_account_id }}' -AND credentials_name = '{{ credentials_name }}' - -/*+ create */ -INSERT INTO databricks_account.provisioning.credentials ( -account_id, -data__credentials_name, -data__aws_credentials -) -SELECT -'{{ databricks_account_id }}', -'{{ credentials_name }}', -'{{ aws_credentials }}' - -/*+ exports, retries=3, retry_delay=5 */ -SELECT -'{{ credentials_name }}' as databricks_credentials_name, -credentials_id as databricks_credentials_id, -JSON_EXTRACT(aws_credentials, '$.sts_role.external_id') as databricks_role_external_id -FROM databricks_account.provisioning.credentials -WHERE account_id = '{{ databricks_account_id }}' -AND credentials_name = '{{ credentials_name }}' -AND JSON_EXTRACT(aws_credentials, '$.sts_role.role_arn') = '{{ aws_iam_cross_account_role_arn }}' - -/*+ delete */ -DELETE FROM databricks_account.provisioning.credentials -WHERE account_id = '{{ databricks_account_id }}' AND -credentials_id = '{{ databricks_credentials_id }}'; \ No newline at end of file diff --git a/examples/databricks/serverless-old/resources/databricks_account/get_users.iql b/examples/databricks/serverless-old/resources/databricks_account/get_users.iql deleted file mode 100644 index e94c2d7..0000000 --- a/examples/databricks/serverless-old/resources/databricks_account/get_users.iql +++ /dev/null @@ -1,6 +0,0 @@ -/*+ exports, retries=3, retry_delay=5 */ -SELECT -JSON_GROUP_ARRAY(JSON_OBJECT('value', id)) as databricks_workspace_group_members -FROM databricks_account.iam.users -WHERE account_id = '{{ databricks_account_id }}' -AND userName in {{ users | sql_list }}; \ No newline at end of file diff --git a/examples/databricks/serverless-old/resources/databricks_account/network.iql b/examples/databricks/serverless-old/resources/databricks_account/network.iql deleted file mode 100644 index fca4c98..0000000 --- a/examples/databricks/serverless-old/resources/databricks_account/network.iql +++ /dev/null @@ -1,41 +0,0 @@ -/*+ exists */ -SELECT COUNT(*) as count -FROM databricks_account.provisioning.networks -WHERE account_id = '{{ databricks_account_id }}' -AND network_name = '{{ databricks_network_name }}' - -/*+ create */ -INSERT INTO databricks_account.provisioning.networks ( -account_id, -data__network_name, -data__vpc_id, -data__subnet_ids, -data__security_group_ids -) -SELECT -'{{ databricks_account_id }}', -'{{ databricks_network_name }}', -'{{ vpc_id }}', -'{{ subnet_ids }}', -'{{ security_group_ids }}' - -/*+ exports, retries=3, retry_delay=5 */ -SELECT -network_id as databricks_network_id -FROM ( -SELECT -network_id, -JSON_EQUAL(subnet_ids, '{{ subnet_ids }}') as subnet_test, -JSON_EQUAL(security_group_ids, '{{ security_group_ids }}') as sg_test -FROM databricks_account.provisioning.networks -WHERE account_id = '{{ databricks_account_id }}' -AND network_name = '{{ databricks_network_name }}' -AND vpc_id = '{{ vpc_id }}' -)t -WHERE subnet_test = 1 -AND sg_test = 1 - -/*+ delete */ -DELETE FROM databricks_account.provisioning.networks -WHERE account_id = '{{ databricks_account_id }}' AND -network_id = '{{ databricks_network_id }}' \ No newline at end of file diff --git a/examples/databricks/serverless-old/resources/databricks_account/storage_configuration.iql b/examples/databricks/serverless-old/resources/databricks_account/storage_configuration.iql deleted file mode 100644 index 8318df1..0000000 --- a/examples/databricks/serverless-old/resources/databricks_account/storage_configuration.iql +++ /dev/null @@ -1,29 +0,0 @@ -/*+ exists */ -SELECT COUNT(*) as count -FROM databricks_account.provisioning.storage -WHERE account_id = '{{ databricks_account_id }}' -AND storage_configuration_name = '{{ storage_configuration_name }}' - -/*+ create */ -INSERT INTO databricks_account.provisioning.storage ( -account_id, -data__storage_configuration_name, -data__root_bucket_info -) -SELECT -'{{ databricks_account_id }}', -'{{ storage_configuration_name }}', -'{{ root_bucket_info }}' - -/*+ exports, retries=3, retry_delay=5 */ -SELECT -storage_configuration_id as databricks_storage_configuration_id -FROM databricks_account.provisioning.storage -WHERE account_id = '{{ databricks_account_id }}' -AND storage_configuration_name = '{{ storage_configuration_name }}' -AND JSON_EXTRACT(root_bucket_info, '$.bucket_name') = '{{ aws_s3_workspace_bucket_name }}' - -/*+ delete */ -DELETE FROM databricks_account.provisioning.storage -WHERE account_id = '{{ databricks_account_id }}' AND -storage_configuration_id = '{{ databricks_storage_configuration_id }}' \ No newline at end of file diff --git a/examples/databricks/serverless-old/resources/databricks_account/update_group_membership.iql b/examples/databricks/serverless-old/resources/databricks_account/update_group_membership.iql deleted file mode 100644 index 375d926..0000000 --- a/examples/databricks/serverless-old/resources/databricks_account/update_group_membership.iql +++ /dev/null @@ -1,6 +0,0 @@ -/*+ command */ -update databricks_account.iam.groups -set data__schemas = '["urn:ietf:params:scim:api:messages:2.0:PatchOp"]', -data__Operations = '[{"op": "replace", "path": "members", "value": {{ databricks_workspace_group_members }} }]' -WHERE account_id = '{{ databricks_account_id }}' -AND id = '{{ databricks_group_id }}'; diff --git a/examples/databricks/serverless-old/resources/databricks_account/workspace.iql b/examples/databricks/serverless-old/resources/databricks_account/workspace.iql deleted file mode 100644 index 1a7efc1..0000000 --- a/examples/databricks/serverless-old/resources/databricks_account/workspace.iql +++ /dev/null @@ -1,42 +0,0 @@ -/*+ exists */ -SELECT COUNT(*) as count -FROM databricks_account.provisioning.workspaces -WHERE account_id = '{{ databricks_account_id }}' -AND workspace_name = '{{ workspace_name }}' - -/*+ create */ -INSERT INTO databricks_account.provisioning.workspaces ( -account_id, -data__workspace_name, -data__aws_region, -data__credentials_id, -data__storage_configuration_id, -data__pricing_tier -) -SELECT -'{{ databricks_account_id }}', -'{{ workspace_name }}', -'{{ aws_region }}', -'{{ credentials_id }}', -'{{ storage_configuration_id }}', -'{{ pricing_tier }}' - -/*+ exports, retries=3, retry_delay=5 */ -SELECT -'{{ workspace_name }}' AS databricks_workspace_name, -workspace_id AS databricks_workspace_id, -deployment_name AS databricks_deployment_name, -workspace_status AS databricks_workspace_status, -'https://' || deployment_name || '.cloud.databricks.com' AS databricks_workspace_url -FROM databricks_account.provisioning.workspaces -WHERE account_id = '{{ databricks_account_id }}' -AND workspace_name = '{{ workspace_name }}' -AND aws_region = '{{ aws_region }}' -AND credentials_id = '{{ credentials_id }}' -AND storage_configuration_id = '{{ storage_configuration_id }}' -AND pricing_tier = '{{ pricing_tier }}' - -/*+ delete */ -DELETE FROM databricks_account.provisioning.workspaces -WHERE account_id = '{{ databricks_account_id }}' AND -workspace_id = '{{ databricks_workspace_id }}' \ No newline at end of file diff --git a/examples/databricks/serverless-old/resources/databricks_account/workspace_group.iql b/examples/databricks/serverless-old/resources/databricks_account/workspace_group.iql deleted file mode 100644 index d2d0d13..0000000 --- a/examples/databricks/serverless-old/resources/databricks_account/workspace_group.iql +++ /dev/null @@ -1,26 +0,0 @@ -/*+ exists */ -SELECT COUNT(*) as count -FROM databricks_account.iam.groups -WHERE account_id = '{{ databricks_account_id }}' -AND displayName = '{{ display_name }}' - -/*+ create */ -INSERT INTO databricks_account.iam.groups ( -account_id, -data__displayName -) -SELECT -'{{ databricks_account_id }}', -'{{ display_name }}' - -/*+ exports, retries=3, retry_delay=5 */ -SELECT id AS databricks_group_id, -displayName AS databricks_group_name -FROM databricks_account.iam.groups -WHERE account_id = '{{ databricks_account_id }}' -AND displayName = '{{ display_name }}' - -/*+ delete */ -DELETE FROM databricks_account.iam.groups -WHERE account_id = '{{ databricks_account_id }}' AND -id = '{{ databricks_group_id }}'; \ No newline at end of file diff --git a/examples/databricks/serverless-old/resources/databricks_account/workspace_permission_assignments.iql b/examples/databricks/serverless-old/resources/databricks_account/workspace_permission_assignments.iql deleted file mode 100644 index 00387e3..0000000 --- a/examples/databricks/serverless-old/resources/databricks_account/workspace_permission_assignments.iql +++ /dev/null @@ -1,32 +0,0 @@ -/*+ exists */ -SELECT COUNT(*) as count -FROM databricks_account.iam.workspace_permission_assignments -WHERE account_id = '{{ databricks_account_id }}' AND -workspace_id = '{{ databricks_workspace_id }}' -AND JSON_EXTRACT(principal, '$.principal_id') = {{ databricks_group_id }} - -/*+ createorupdate */ -INSERT INTO databricks_account.iam.workspace_permission_assignments ( -account_id, -principal_id, -workspace_id, -data__permissions -) -SELECT -'{{ databricks_account_id }}', -'{{ databricks_group_id }}', -'{{ databricks_workspace_id }}', -'["ADMIN"]' - -/*+ statecheck, retries=3, retry_delay=5 */ -SELECT COUNT(*) as count -FROM databricks_account.iam.workspace_permission_assignments -WHERE account_id = '{{ databricks_account_id }}' AND -workspace_id = '{{ databricks_workspace_id }}' -AND JSON_EXTRACT(principal, '$.principal_id') = {{ databricks_group_id }} - -/*+ delete */ -DELETE FROM databricks_account.iam.workspace_permission_assignments -WHERE account_id = '{{ databricks_account_id }}' AND -principal_id = '{{ databricks_group_id }}' AND -workspace_id = '{{ databricks_workspace_id }}' \ No newline at end of file diff --git a/examples/databricks/serverless-old/resources/databricks_workspace/external_location.iql b/examples/databricks/serverless-old/resources/databricks_workspace/external_location.iql deleted file mode 100644 index 4d993d0..0000000 --- a/examples/databricks/serverless-old/resources/databricks_workspace/external_location.iql +++ /dev/null @@ -1,40 +0,0 @@ -/*+ exists */ -SELECT COUNT(*) as count -FROM databricks_workspace.unitycatalog.external_locations -WHERE name = '{{ name | replace('-', '_') }}' AND -deployment_name = '{{ databricks_deployment_name }}'; - -/*+ create */ -INSERT INTO databricks_workspace.unitycatalog.external_locations ( -deployment_name, -data__name, -data__url, -data__credential_name, -data__read_only, -data__comment, -data__skip_validation -) -SELECT -'{{ databricks_deployment_name }}', -'{{ name | replace('-', '_') }}', -'{{ url }}', -'{{ credential_name | replace('-', '_') }}', -{{ read_only }}, -'{{ comment }}', -{{ skip_validation }} -; - -/*+ exports, retries=3, retry_delay=5 */ -SELECT name as external_location_name -FROM databricks_workspace.unitycatalog.external_locations -WHERE name = '{{ name | replace('-', '_') }}' AND -deployment_name = '{{ databricks_deployment_name }}' -AND url = '{{ url }}' AND -credential_name = '{{ credential_name | replace('-', '_') }}' AND -read_only = {{ read_only }} AND -comment = '{{ comment }}'; - -/*+ delete */ -DELETE FROM databricks_workspace.unitycatalog.external_locations -WHERE name = '{{ name | replace('-', '_') }}' AND -deployment_name = '{{ databricks_deployment_name }}'; \ No newline at end of file diff --git a/examples/databricks/serverless-old/resources/databricks_workspace/storage_credential.iql b/examples/databricks/serverless-old/resources/databricks_workspace/storage_credential.iql deleted file mode 100644 index 65dd110..0000000 --- a/examples/databricks/serverless-old/resources/databricks_workspace/storage_credential.iql +++ /dev/null @@ -1,37 +0,0 @@ -/*+ exists */ -SELECT COUNT(*) as count -FROM databricks_workspace.unitycatalog.storage_credentials -WHERE name = '{{ name | replace('-', '_') | upper }}' AND -deployment_name = '{{ databricks_deployment_name }}'; - -/*+ create */ -INSERT INTO databricks_workspace.unitycatalog.storage_credentials ( -deployment_name, -data__name, -data__comment, -data__read_only, -data__aws_iam_role, -data__skip_validation -) -SELECT -'{{ databricks_deployment_name }}', -'{{ name | replace('-', '_') | upper }}', -'{{ comment }}', -'{{ read_only }}', -'{{ aws_iam_role }}', -'{{ skip_validation }}' -; - -/*+ exports, retries=3, retry_delay=5 */ -SELECT -name as storage_credential_name, -JSON_EXTRACT(aws_iam_role, '$.external_id') as storage_credential_external_id -FROM databricks_workspace.unitycatalog.storage_credentials -WHERE name = '{{ name | replace('-', '_') | upper }}' AND -deployment_name = '{{ databricks_deployment_name }}' AND -JSON_EXTRACT(aws_iam_role, '$.role_arn') = '{{ metastore_access_role_arn }}'; - -/*+ delete */ -DELETE FROM databricks_workspace.unitycatalog.storage_credentials -WHERE name = '{{ name | replace('-', '_') | upper }}' AND -deployment_name = '{{ databricks_deployment_name }}'; \ No newline at end of file diff --git a/examples/databricks/serverless-old/sec/.gitignore b/examples/databricks/serverless-old/sec/.gitignore deleted file mode 100644 index d6b7ef3..0000000 --- a/examples/databricks/serverless-old/sec/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/examples/databricks/serverless-old/stackql_manifest.yml b/examples/databricks/serverless-old/stackql_manifest.yml deleted file mode 100644 index ab20021..0000000 --- a/examples/databricks/serverless-old/stackql_manifest.yml +++ /dev/null @@ -1,497 +0,0 @@ -version: 1 -name: "stackql-serverless" -description: creates a serverless databricks workspace -providers: - - aws - - databricks_account - - databricks_workspace -globals: - - name: databricks_account_id - description: databricks account id - value: "{{ DATABRICKS_ACCOUNT_ID }}" - - name: databricks_aws_account_id - description: databricks AWS account id - value: "{{ DATABRICKS_AWS_ACCOUNT_ID }}" - - name: aws_account - description: aws_account id - value: "{{ AWS_ACCOUNT_ID }}" - - name: region - description: aws region - value: "{{ AWS_REGION }}" - - name: global_tags - value: - - Key: 'stackql:stack-name' - Value: "{{ stack_name }}" - - Key: 'stackql:stack-env' - Value: "{{ stack_env }}" -resources: - -# ==================================================================================== -# IAM and Cloud Credentials -# ==================================================================================== - - - name: aws/iam/cross_account_role - file: aws/iam/iam_role.iql - props: - - name: role_name - value: "{{ stack_name }}-{{ stack_env }}-role" - - name: assume_role_policy_document - value: - Version: "2012-10-17" - Statement: - - Sid: "" - Effect: "Allow" - Principal: - AWS: "arn:aws:iam::{{ databricks_aws_account_id }}:root" - Action: "sts:AssumeRole" - Condition: - StringEquals: - sts:ExternalId: "{{ databricks_account_id }}" - - name: description - value: 'allows Databricks to access resources in ({{ stack_name }}-{{ stack_env }})' - - name: path - value: '/' - - name: policies - value: - - PolicyDocument: - Statement: - - Sid: Stmt1403287045000 - Effect: Allow - Action: - - "ec2:AllocateAddress" - - "ec2:AssociateDhcpOptions" - - "ec2:AssociateIamInstanceProfile" - - "ec2:AssociateRouteTable" - - "ec2:AttachInternetGateway" - - "ec2:AttachVolume" - - "ec2:AuthorizeSecurityGroupEgress" - - "ec2:AuthorizeSecurityGroupIngress" - - "ec2:CancelSpotInstanceRequests" - - "ec2:CreateDhcpOptions" - - "ec2:CreateInternetGateway" - - "ec2:CreateKeyPair" - - "ec2:CreateNatGateway" - - "ec2:CreatePlacementGroup" - - "ec2:CreateRoute" - - "ec2:CreateRouteTable" - - "ec2:CreateSecurityGroup" - - "ec2:CreateSubnet" - - "ec2:CreateTags" - - "ec2:CreateVolume" - - "ec2:CreateVpc" - - "ec2:CreateVpcEndpoint" - - "ec2:DeleteDhcpOptions" - - "ec2:DeleteInternetGateway" - - "ec2:DeleteKeyPair" - - "ec2:DeleteNatGateway" - - "ec2:DeletePlacementGroup" - - "ec2:DeleteRoute" - - "ec2:DeleteRouteTable" - - "ec2:DeleteSecurityGroup" - - "ec2:DeleteSubnet" - - "ec2:DeleteTags" - - "ec2:DeleteVolume" - - "ec2:DeleteVpc" - - "ec2:DeleteVpcEndpoints" - - "ec2:DescribeAvailabilityZones" - - "ec2:DescribeIamInstanceProfileAssociations" - - "ec2:DescribeInstanceStatus" - - "ec2:DescribeInstances" - - "ec2:DescribeInternetGateways" - - "ec2:DescribeNatGateways" - - "ec2:DescribePlacementGroups" - - "ec2:DescribePrefixLists" - - "ec2:DescribeReservedInstancesOfferings" - - "ec2:DescribeRouteTables" - - "ec2:DescribeSecurityGroups" - - "ec2:DescribeSpotInstanceRequests" - - "ec2:DescribeSpotPriceHistory" - - "ec2:DescribeSubnets" - - "ec2:DescribeVolumes" - - "ec2:DescribeVpcs" - - "ec2:DescribeVpcAttribute" - - "ec2:DescribeNetworkAcls" - - "ec2:DetachInternetGateway" - - "ec2:DisassociateIamInstanceProfile" - - "ec2:DisassociateRouteTable" - - "ec2:ModifyVpcAttribute" - - "ec2:ReleaseAddress" - - "ec2:ReplaceIamInstanceProfileAssociation" - - "ec2:ReplaceRoute" - - "ec2:RequestSpotInstances" - - "ec2:RevokeSecurityGroupEgress" - - "ec2:RevokeSecurityGroupIngress" - - "ec2:RunInstances" - - "ec2:TerminateInstances" - Resource: - - "*" - - Effect: Allow - Action: - - "iam:CreateServiceLinkedRole" - - "iam:PutRolePolicy" - Resource: - - arn:aws:iam::*:role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot - Condition: - StringLike: - "iam:AWSServiceName": spot.amazonaws.com - Version: '2012-10-17' - PolicyName: "{{ stack_name }}-{{ stack_env }}-policy" - exports: - - aws_iam_role_name: aws_iam_cross_account_role_name - - aws_iam_role_arn: aws_iam_cross_account_role_arn - - - name: databricks_account/credentials - props: - - name: credentials_name - value: "{{ stack_name }}-{{ stack_env }}-credentials" - - name: aws_credentials - value: - sts_role: - role_arn: "{{ aws_iam_cross_account_role_arn }}" - exports: - - databricks_credentials_name - - databricks_credentials_id - - databricks_role_external_id - -# ==================================================================================== -# Storage -# ==================================================================================== - - - name: aws/s3/workspace_bucket - file: aws/s3/s3_bucket.iql - props: - - name: bucket_name - value: "{{ stack_name }}-{{ stack_env }}-root-bucket" - - name: ownership_controls - value: - Rules: - - ObjectOwnership: "BucketOwnerPreferred" - - name: bucket_encryption - value: - ServerSideEncryptionConfiguration: - - BucketKeyEnabled: true - ServerSideEncryptionByDefault: - SSEAlgorithm: "AES256" - - name: public_access_block_configuration - value: - BlockPublicAcls: true - IgnorePublicAcls: true - BlockPublicPolicy: true - RestrictPublicBuckets: true - - name: versioning_configuration - value: - Status: "Suspended" - exports: - - arn: aws_s3_workspace_bucket_arn - - bucket_name: aws_s3_workspace_bucket_name - - - name: aws/s3/workspace_bucket_policy - file: aws/s3/s3_bucket_policy.iql - props: - - name: policy_document - value: - Version: "2012-10-17" - Statement: - - Sid: Grant Databricks Access - Effect: Allow - Principal: - AWS: "arn:aws:iam::{{ databricks_aws_account_id }}:root" - Action: - - "s3:GetObject" - - "s3:GetObjectVersion" - - "s3:PutObject" - - "s3:DeleteObject" - - "s3:ListBucket" - - "s3:GetBucketLocation" - Resource: - - "{{ aws_s3_workspace_bucket_arn }}/*" - - "{{ aws_s3_workspace_bucket_arn }}" - - - name: databricks_account/storage_configuration - props: - - name: storage_configuration_name - value: "{{ stack_name }}-{{ stack_env }}-storage" - - name: root_bucket_info - value: - bucket_name: "{{ aws_s3_workspace_bucket_name }}" - exports: - - databricks_storage_configuration_id - -# ==================================================================================== -# UC Storage Credential and Metastore Catalog Bucket -# ==================================================================================== - - - name: aws/s3/metastore_bucket - file: aws/s3/s3_bucket.iql - props: - - name: bucket_name - value: "{{ stack_name }}-{{ stack_env }}-metastore" - - name: ownership_controls - value: - Rules: - - ObjectOwnership: "BucketOwnerPreferred" - - name: bucket_encryption - value: - ServerSideEncryptionConfiguration: - - BucketKeyEnabled: true - ServerSideEncryptionByDefault: - SSEAlgorithm: "AES256" - - name: public_access_block_configuration - value: - BlockPublicAcls: true - IgnorePublicAcls: true - BlockPublicPolicy: true - RestrictPublicBuckets: true - - name: versioning_configuration - value: - Status: "Suspended" - exports: - - arn: aws_s3_metastore_bucket_arn - - bucket_name: aws_s3_metastore_bucket_name - - - name: aws/iam/metastore_access_role - file: aws/iam/iam_role.iql - props: - - name: role_name - value: "{{ stack_name }}-{{ stack_env }}-metastore-role" - - name: assume_role_policy_document - value: - Version: "2012-10-17" - Statement: - - Effect: "Allow" - Principal: - AWS: - - "arn:aws:iam::414351767826:role/unity-catalog-prod-UCMasterRole-14S5ZJVKOTYTL" - Action: "sts:AssumeRole" - Condition: - StringEquals: - sts:ExternalId: "0000" # Placeholder - - name: description - value: 'Unity Catalog metastore access role for ({{ stack_name }}-{{ stack_env }})' - - name: path - value: '/' - - name: policies - value: - - PolicyName: "MetastoreS3Policy" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: "Allow" - Action: - - "s3:GetObject" - - "s3:PutObject" - - "s3:DeleteObject" - - "s3:ListBucket" - - "s3:GetBucketLocation" - - "s3:ListBucketMultipartUploads" - - "s3:ListMultipartUploadParts" - - "s3:AbortMultipartUpload" - Resource: - - "{{ aws_s3_metastore_bucket_arn }}/*" - - "{{ aws_s3_metastore_bucket_arn }}" - - # - Effect: "Allow" - # Action: - # - "kms:Decrypt" - # - "kms:Encrypt" - # - "kms:GenerateDataKey*" - # Resource: - # - "arn:aws:kms:" - - - Effect: "Allow" - Action: - - "sts:AssumeRole" - Resource: - - "arn:aws:iam::{{ databricks_aws_account_id }}:role/{{ stack_name }}-{{ stack_env }}-metastore-role" - - - Sid: "ManagedFileEventsSetupStatement" - Effect: "Allow" - Action: - - "s3:GetBucketNotification" - - "s3:PutBucketNotification" - - "sns:ListSubscriptionsByTopic" - - "sns:GetTopicAttributes" - - "sns:SetTopicAttributes" - - "sns:CreateTopic" - - "sns:TagResource" - - "sns:Publish" - - "sns:Subscribe" - - "sqs:CreateQueue" - - "sqs:DeleteMessage" - - "sqs:ReceiveMessage" - - "sqs:SendMessage" - - "sqs:GetQueueUrl" - - "sqs:GetQueueAttributes" - - "sqs:SetQueueAttributes" - - "sqs:TagQueue" - - "sqs:ChangeMessageVisibility" - - "sqs:PurgeQueue" - Resource: - - "{{ aws_s3_metastore_bucket_arn }}" - - "arn:aws:sqs:*:*:csms-*" - - "arn:aws:sns:*:*:csms-*" - - - Sid: "ManagedFileEventsListStatement" - Effect: "Allow" - Action: - - "sqs:ListQueues" - - "sqs:ListQueueTags" - - "sns:ListTopics" - Resource: - - "arn:aws:sqs:*:*:csms-*" - - "arn:aws:sns:*:*:csms-*" - - - Sid: "ManagedFileEventsTeardownStatement" - Effect: "Allow" - Action: - - "sns:Unsubscribe" - - "sns:DeleteTopic" - - "sqs:DeleteQueue" - Resource: - - "arn:aws:sqs:*:*:csms-*" - - "arn:aws:sns:*:*:csms-*" - - name: tags - value: - - Key: Purpose - Value: "Unity Catalog Storage Credential" - merge: - - global_tags - skip_validation: true - exports: - - aws_iam_role_arn: metastore_access_role_arn - -# ==================================================================================== -# DBX Workspace -# ==================================================================================== - - - name: databricks_account/workspace - props: - - name: workspace_name - value: "{{ stack_name }}-{{ stack_env }}-workspace" - - name: aws_region - value: "{{ region }}" - - name: credentials_id - value: "{{ databricks_credentials_id }}" - - name: storage_configuration_id - value: "{{ databricks_storage_configuration_id }}" - - name: pricing_tier - value: PREMIUM - exports: - - databricks_workspace_name - - databricks_workspace_id - - databricks_deployment_name - - databricks_workspace_status - - databricks_workspace_url - - - name: databricks_account/workspace_group - props: - - name: display_name - value: "{{ stack_name }}-{{ stack_env }}-workspace-admins" - exports: - - databricks_group_id - - databricks_group_name - - - name: databricks_account/get_users - type: query - props: - - name: users - value: - - "javen@stackql.io" - - "krimmer@stackql.io" - exports: - - databricks_workspace_group_members - - - name: databricks_account/update_group_membership - type: command - props: [] - - - name: databricks_account/workspace_permission_assignments - props: [] - - - name: databricks_workspace/storage_credential - props: - - name: name - value: "{{ stack_name }}_{{ stack_env }}_storage_credential" - - name: comment - value: "Storage credential for {{ stack_name }} {{ stack_env }} metastore S3 access" - - name: read_only - value: false - - name: aws_iam_role - value: - role_arn: "{{ metastore_access_role_arn }}" - - name: skip_validation - value: false - exports: - - storage_credential_name - - storage_credential_external_id - - - name: aws/iam/update_metastore_access_role - type: command - props: - - name: role_name - value: "{{ stack_name }}-{{ stack_env }}-metastore-role" - - name: assume_role_policy_document - value: - Version: "2012-10-17" - Statement: - - Effect: "Allow" - Principal: - AWS: - - "arn:aws:iam::414351767826:role/unity-catalog-prod-UCMasterRole-14S5ZJVKOTYTL" - - "{{ metastore_access_role_arn }}" - Action: "sts:AssumeRole" - Condition: - StringEquals: - sts:ExternalId: "{{ storage_credential_external_id }}" - - - name: databricks_workspace/unitycatalog/credential_grants - type: command - props: - - name: privileges - value: - - "ALL_PRIVILEGES" - - "MANAGE" - sql: | - UPDATE databricks_workspace.unitycatalog.grants - SET data__changes = '[{"add": {{ privileges }},"principal": "{{ databricks_group_name }}"}]' - WHERE full_name = '{{ storage_credential_name }}' AND - securable_type = 'storage_credential' AND - deployment_name = '{{ databricks_deployment_name }}'; - - - name: databricks_workspace/external_location - props: - - name: name - value: "{{ stack_name }}_{{ stack_env }}_external_location" - - name: comment - value: "External location for {{ stack_name }} {{ stack_env }} metastore S3 access" - - name: url - value: "s3://{{ aws_s3_metastore_bucket_name }}/unitycatalog/demo" - - name: credential_name - value: "{{ storage_credential_name }}" - - name: read_only - value: false - - name: skip_validation - value: false - exports: - - external_location_name - - - name: databricks_workspace/unitycatalog/location_grants - type: command - props: - - name: privileges - value: - - "ALL_PRIVILEGES" - - "MANAGE" - sql: | - UPDATE databricks_workspace.unitycatalog.grants - SET data__changes = '[{"add": {{ privileges }},"principal": "{{ databricks_group_name }}"}]' - WHERE full_name = '{{ external_location_name }}' AND - securable_type = 'external_location' AND - deployment_name = '{{ databricks_deployment_name }}'; - -exports: - - databricks_workspace_name - - databricks_workspace_id - - databricks_deployment_name - - databricks_workspace_status - - databricks_workspace_url \ No newline at end of file diff --git a/examples/databricks/serverless/resources/OLD/aws/iam/iam_role.iql b/examples/databricks/serverless/resources/OLD/aws/iam/iam_role.iql index 0e7d3a0..ff01932 100644 --- a/examples/databricks/serverless/resources/OLD/aws/iam/iam_role.iql +++ b/examples/databricks/serverless/resources/OLD/aws/iam/iam_role.iql @@ -50,7 +50,7 @@ SELECT COUNT(*) as count FROM ( AWS_POLICY_EQUAL(assume_role_policy_document, '{{ assume_role_policy_document }}') as test_assume_role_policy_doc, AWS_POLICY_EQUAL(policies, '{{ policies }}') as test_policies FROM aws.iam.roles - WHERE data__Identifier = '{{ role_name }}')t + WHERE Identifier = '{{ role_name }}')t WHERE test_assume_role_policy_doc = 1 AND test_policies = 1 AND path = '{{ path }}'; @@ -60,7 +60,7 @@ SELECT '{{ role_name }}' as aws_iam_role_name, arn as aws_iam_role_arn FROM aws.iam.roles -WHERE data__Identifier = '{{ role_name }}' +WHERE Identifier = '{{ role_name }}' /*+ delete */ DELETE FROM awscc.iam.roles diff --git a/examples/databricks/serverless/resources/OLD/aws/iam/update_metastore_access_role.iql b/examples/databricks/serverless/resources/OLD/aws/iam/update_metastore_access_role.iql index 2339232..0bc750a 100644 --- a/examples/databricks/serverless/resources/OLD/aws/iam/update_metastore_access_role.iql +++ b/examples/databricks/serverless/resources/OLD/aws/iam/update_metastore_access_role.iql @@ -1,7 +1,7 @@ /*+ command */ update aws.iam.roles -set data__PatchDocument = string('{{ { +set PatchDocument = string('{{ { "AssumeRolePolicyDocument": assume_role_policy_document } | generate_patch_document }}') -WHERE data__Identifier = '{{ role_name }}' +WHERE Identifier = '{{ role_name }}' AND region = 'us-east-1'; \ No newline at end of file diff --git a/examples/databricks/serverless/resources/OLD/aws/s3/s3_bucket.iql b/examples/databricks/serverless/resources/OLD/aws/s3/s3_bucket.iql index 42741fd..48baabe 100644 --- a/examples/databricks/serverless/resources/OLD/aws/s3/s3_bucket.iql +++ b/examples/databricks/serverless/resources/OLD/aws/s3/s3_bucket.iql @@ -2,7 +2,7 @@ SELECT COUNT(*) as count FROM aws.s3.buckets WHERE region = '{{ region }}' -AND data__Identifier = '{{ bucket_name }}' +AND Identifier = '{{ bucket_name }}' /*+ create */ INSERT INTO aws.s3.buckets ( @@ -25,7 +25,7 @@ SELECT /*+ update */ update aws.s3.buckets -set data__PatchDocument = string('{{ { +set PatchDocument = string('{{ { "OwnershipControls": ownership_controls, "BucketEncryption": bucket_encryption, "PublicAccessBlockConfiguration": public_access_block_configuration, @@ -34,7 +34,7 @@ set data__PatchDocument = string('{{ { } | generate_patch_document }}') WHERE region = '{{ region }}' -AND data__Identifier = '{{ bucket_name }}' +AND Identifier = '{{ bucket_name }}' /*+ exports, retries=3, retry_delay=5 */ SELECT @@ -50,7 +50,7 @@ FROM ( JSON_EQUAL(versioning_configuration, '{{ versioning_configuration }}') as test_versioning_configuration FROM aws.s3.buckets WHERE region = '{{ region }}' - AND data__Identifier = '{{ bucket_name }}' + AND Identifier = '{{ bucket_name }}' )t WHERE test_ownership_controls = 1 AND test_encryption = 1 diff --git a/examples/databricks/serverless/resources/OLD/aws/s3/s3_bucket_policy.iql b/examples/databricks/serverless/resources/OLD/aws/s3/s3_bucket_policy.iql index 69ec36b..dc6a724 100644 --- a/examples/databricks/serverless/resources/OLD/aws/s3/s3_bucket_policy.iql +++ b/examples/databricks/serverless/resources/OLD/aws/s3/s3_bucket_policy.iql @@ -19,12 +19,12 @@ SELECT /*+ update */ update aws.s3.bucket_policies -set data__PatchDocument = string('{{ { +set PatchDocument = string('{{ { "PolicyDocument": policy_document } | generate_patch_document }}') WHERE region = '{{ region }}' -AND data__Identifier = '{{ aws_s3_workspace_bucket_name }}'; +AND Identifier = '{{ aws_s3_workspace_bucket_name }}'; /*+ statecheck, retries=3, retry_delay=5 */ SELECT COUNT(*) as count FROM ( @@ -32,5 +32,5 @@ SELECT COUNT(*) as count FROM ( AWS_POLICY_EQUAL(policy_document, '{{ policy_document }}') as test_policy_document FROM aws.s3.bucket_policies WHERE region = '{{ region }}' - AND data__Identifier = '{{ aws_s3_workspace_bucket_name }}')t + AND Identifier = '{{ aws_s3_workspace_bucket_name }}')t WHERE test_policy_document = 1; diff --git a/examples/databricks/serverless/resources/OLD/databricks_account/credentials.iql b/examples/databricks/serverless/resources/OLD/databricks_account/credentials.iql index 687b3f1..8c841c1 100644 --- a/examples/databricks/serverless/resources/OLD/databricks_account/credentials.iql +++ b/examples/databricks/serverless/resources/OLD/databricks_account/credentials.iql @@ -7,8 +7,8 @@ AND credentials_name = '{{ credentials_name }}' /*+ create */ INSERT INTO databricks_account.provisioning.credentials ( account_id, -data__credentials_name, -data__aws_credentials +credentials_name, +aws_credentials ) SELECT '{{ databricks_account_id }}', diff --git a/examples/databricks/serverless/resources/OLD/databricks_account/network.iql b/examples/databricks/serverless/resources/OLD/databricks_account/network.iql index fca4c98..541e3ed 100644 --- a/examples/databricks/serverless/resources/OLD/databricks_account/network.iql +++ b/examples/databricks/serverless/resources/OLD/databricks_account/network.iql @@ -7,10 +7,10 @@ AND network_name = '{{ databricks_network_name }}' /*+ create */ INSERT INTO databricks_account.provisioning.networks ( account_id, -data__network_name, -data__vpc_id, -data__subnet_ids, -data__security_group_ids +network_name, +vpc_id, +subnet_ids, +security_group_ids ) SELECT '{{ databricks_account_id }}', diff --git a/examples/databricks/serverless/resources/OLD/databricks_account/storage_configuration.iql b/examples/databricks/serverless/resources/OLD/databricks_account/storage_configuration.iql index 8318df1..c118b4e 100644 --- a/examples/databricks/serverless/resources/OLD/databricks_account/storage_configuration.iql +++ b/examples/databricks/serverless/resources/OLD/databricks_account/storage_configuration.iql @@ -7,8 +7,8 @@ AND storage_configuration_name = '{{ storage_configuration_name }}' /*+ create */ INSERT INTO databricks_account.provisioning.storage ( account_id, -data__storage_configuration_name, -data__root_bucket_info +storage_configuration_name, +root_bucket_info ) SELECT '{{ databricks_account_id }}', diff --git a/examples/databricks/serverless/resources/OLD/databricks_account/update_group_membership.iql b/examples/databricks/serverless/resources/OLD/databricks_account/update_group_membership.iql index 375d926..7f28c52 100644 --- a/examples/databricks/serverless/resources/OLD/databricks_account/update_group_membership.iql +++ b/examples/databricks/serverless/resources/OLD/databricks_account/update_group_membership.iql @@ -1,6 +1,6 @@ /*+ command */ update databricks_account.iam.groups -set data__schemas = '["urn:ietf:params:scim:api:messages:2.0:PatchOp"]', -data__Operations = '[{"op": "replace", "path": "members", "value": {{ databricks_workspace_group_members }} }]' +set schemas = '["urn:ietf:params:scim:api:messages:2.0:PatchOp"]', +Operations = '[{"op": "replace", "path": "members", "value": {{ databricks_workspace_group_members }} }]' WHERE account_id = '{{ databricks_account_id }}' AND id = '{{ databricks_group_id }}'; diff --git a/examples/databricks/serverless/resources/OLD/databricks_account/workspace.iql b/examples/databricks/serverless/resources/OLD/databricks_account/workspace.iql index 1a7efc1..1094860 100644 --- a/examples/databricks/serverless/resources/OLD/databricks_account/workspace.iql +++ b/examples/databricks/serverless/resources/OLD/databricks_account/workspace.iql @@ -7,11 +7,11 @@ AND workspace_name = '{{ workspace_name }}' /*+ create */ INSERT INTO databricks_account.provisioning.workspaces ( account_id, -data__workspace_name, -data__aws_region, -data__credentials_id, -data__storage_configuration_id, -data__pricing_tier +workspace_name, +aws_region, +credentials_id, +storage_configuration_id, +pricing_tier ) SELECT '{{ databricks_account_id }}', diff --git a/examples/databricks/serverless/resources/OLD/databricks_account/workspace_group.iql b/examples/databricks/serverless/resources/OLD/databricks_account/workspace_group.iql index d2d0d13..4a2cfcc 100644 --- a/examples/databricks/serverless/resources/OLD/databricks_account/workspace_group.iql +++ b/examples/databricks/serverless/resources/OLD/databricks_account/workspace_group.iql @@ -7,7 +7,7 @@ AND displayName = '{{ display_name }}' /*+ create */ INSERT INTO databricks_account.iam.groups ( account_id, -data__displayName +displayName ) SELECT '{{ databricks_account_id }}', diff --git a/examples/databricks/serverless/resources/OLD/databricks_account/workspace_permission_assignments.iql b/examples/databricks/serverless/resources/OLD/databricks_account/workspace_permission_assignments.iql index 00387e3..6051aba 100644 --- a/examples/databricks/serverless/resources/OLD/databricks_account/workspace_permission_assignments.iql +++ b/examples/databricks/serverless/resources/OLD/databricks_account/workspace_permission_assignments.iql @@ -10,7 +10,7 @@ INSERT INTO databricks_account.iam.workspace_permission_assignments ( account_id, principal_id, workspace_id, -data__permissions +permissions ) SELECT '{{ databricks_account_id }}', diff --git a/examples/databricks/serverless/resources/OLD/databricks_workspace/external_location.iql b/examples/databricks/serverless/resources/OLD/databricks_workspace/external_location.iql index 4d993d0..0df1518 100644 --- a/examples/databricks/serverless/resources/OLD/databricks_workspace/external_location.iql +++ b/examples/databricks/serverless/resources/OLD/databricks_workspace/external_location.iql @@ -7,12 +7,12 @@ deployment_name = '{{ databricks_deployment_name }}'; /*+ create */ INSERT INTO databricks_workspace.unitycatalog.external_locations ( deployment_name, -data__name, -data__url, -data__credential_name, -data__read_only, -data__comment, -data__skip_validation +name, +url, +credential_name, +read_only, +comment, +skip_validation ) SELECT '{{ databricks_deployment_name }}', diff --git a/examples/databricks/serverless/resources/OLD/databricks_workspace/storage_credential.iql b/examples/databricks/serverless/resources/OLD/databricks_workspace/storage_credential.iql index 65dd110..2ade455 100644 --- a/examples/databricks/serverless/resources/OLD/databricks_workspace/storage_credential.iql +++ b/examples/databricks/serverless/resources/OLD/databricks_workspace/storage_credential.iql @@ -7,11 +7,11 @@ deployment_name = '{{ databricks_deployment_name }}'; /*+ create */ INSERT INTO databricks_workspace.unitycatalog.storage_credentials ( deployment_name, -data__name, -data__comment, -data__read_only, -data__aws_iam_role, -data__skip_validation +name, +comment, +read_only, +aws_iam_role, +skip_validation ) SELECT '{{ databricks_deployment_name }}', diff --git a/examples/databricks/serverless/stackql_manifest.yml b/examples/databricks/serverless/stackql_manifest.yml index 9f15ce2..d9ff5a2 100644 --- a/examples/databricks/serverless/stackql_manifest.yml +++ b/examples/databricks/serverless/stackql_manifest.yml @@ -347,7 +347,7 @@ resources: # - "MANAGE" # sql: | # UPDATE databricks_workspace.unitycatalog.grants - # SET data__changes = '[{"add": {{ privileges }},"principal": "{{ databricks_group_name }}"}]' + # SET changes = '[{"add": {{ privileges }},"principal": "{{ databricks_group_name }}"}]' # WHERE full_name = '{{ storage_credential_name }}' AND # securable_type = 'storage_credential' AND # deployment_name = '{{ databricks_deployment_name }}'; @@ -378,7 +378,7 @@ resources: # - "MANAGE" # sql: | # UPDATE databricks_workspace.unitycatalog.grants - # SET data__changes = '[{"add": {{ privileges }},"principal": "{{ databricks_group_name }}"}]' + # SET changes = '[{"add": {{ privileges }},"principal": "{{ databricks_group_name }}"}]' # WHERE full_name = '{{ external_location_name }}' AND # securable_type = 'external_location' AND # deployment_name = '{{ databricks_deployment_name }}'; diff --git a/examples/databricks/snowflake-interoperability/resources/databricks_workspace/catalog.iql b/examples/databricks/snowflake-interoperability/resources/databricks_workspace/catalog.iql index 5d7df7e..9a44fe1 100644 --- a/examples/databricks/snowflake-interoperability/resources/databricks_workspace/catalog.iql +++ b/examples/databricks/snowflake-interoperability/resources/databricks_workspace/catalog.iql @@ -1,9 +1,9 @@ /*+ create */ INSERT INTO databricks_workspace.unitycatalog.catalogs ( deployment_name, -data__name, -data__comment, -data__storage_root +name, +comment, +storage_root ) SELECT '{{ databricks_deployment_name }}', diff --git a/examples/databricks/snowflake-interoperability/resources/databricks_workspace/schema.iql b/examples/databricks/snowflake-interoperability/resources/databricks_workspace/schema.iql index b662259..66ec961 100644 --- a/examples/databricks/snowflake-interoperability/resources/databricks_workspace/schema.iql +++ b/examples/databricks/snowflake-interoperability/resources/databricks_workspace/schema.iql @@ -1,9 +1,9 @@ /*+ create */ INSERT INTO databricks_workspace.unitycatalog.schemas ( deployment_name, -data__name, -data__catalog_name, -data__comment +name, +catalog_name, +comment ) SELECT '{{ databricks_deployment_name }}', diff --git a/examples/databricks/snowflake-interoperability/resources/databricks_workspace/service_principal.iql b/examples/databricks/snowflake-interoperability/resources/databricks_workspace/service_principal.iql index 355adee..ab73bcc 100644 --- a/examples/databricks/snowflake-interoperability/resources/databricks_workspace/service_principal.iql +++ b/examples/databricks/snowflake-interoperability/resources/databricks_workspace/service_principal.iql @@ -1,8 +1,8 @@ /*+ create */ INSERT INTO databricks_workspace.iam.service_principals ( deployment_name, -data__displayName, -data__active +displayName, +active ) SELECT '{{ databricks_deployment_name }}', diff --git a/examples/databricks/snowflake-interoperability/resources/snowflake/statement.iql b/examples/databricks/snowflake-interoperability/resources/snowflake/statement.iql index decce8c..29f7648 100644 --- a/examples/databricks/snowflake-interoperability/resources/snowflake/statement.iql +++ b/examples/databricks/snowflake-interoperability/resources/snowflake/statement.iql @@ -1,10 +1,10 @@ /*+ command */ INSERT INTO snowflake.sqlapi.statements ( -data__statement, -data__timeout, -data__database, -data__schema, -data__warehouse, +statement, +timeout, +database, +schema, +warehouse, "User-Agent", endpoint ) diff --git a/examples/databricks/snowflake-interoperability/stackql_manifest.yml b/examples/databricks/snowflake-interoperability/stackql_manifest.yml index c5d0e2c..2ef784f 100644 --- a/examples/databricks/snowflake-interoperability/stackql_manifest.yml +++ b/examples/databricks/snowflake-interoperability/stackql_manifest.yml @@ -83,7 +83,7 @@ resources: props: [] sql: | UPDATE databricks_workspace.unitycatalog.metastores - SET data__external_access_enabled = 'true' + SET external_access_enabled = 'true' WHERE id = '{{ metastore_id }}' AND deployment_name = '{{ databricks_deployment_name }}'; @@ -112,7 +112,7 @@ resources: - "MANAGE" sql: | UPDATE databricks_workspace.unitycatalog.grants - SET data__changes = '[{"add": {{ privileges }},"principal": "{{ databricks_admin_group }}"}]' + SET changes = '[{"add": {{ privileges }},"principal": "{{ databricks_admin_group }}"}]' WHERE full_name = '{{ catalog_name }}' AND securable_type = 'catalog' AND deployment_name = '{{ databricks_deployment_name }}'; @@ -158,7 +158,7 @@ resources: - "USE_SCHEMA" sql: | UPDATE databricks_workspace.unitycatalog.grants - SET data__changes = '[{"add": {{ privileges }},"principal": "{{ service_principal_application_id }}"}]' + SET changes = '[{"add": {{ privileges }},"principal": "{{ service_principal_application_id }}"}]' WHERE full_name = '{{ catalog_name }}' AND securable_type = 'catalog' AND deployment_name = '{{ databricks_deployment_name }}'; diff --git a/examples/google/k8s-the-hard-way/resources/firewalls.iql b/examples/google/k8s-the-hard-way/resources/firewalls.iql index d69607b..66964a4 100644 --- a/examples/google/k8s-the-hard-way/resources/firewalls.iql +++ b/examples/google/k8s-the-hard-way/resources/firewalls.iql @@ -7,11 +7,11 @@ AND name = '{{ fw_name }}' INSERT INTO google.compute.firewalls ( project, - data__name, - data__network, - data__direction, - data__sourceRanges, - data__allowed + name, + network, + direction, + sourceRanges, + allowed ) SELECT '{{ project }}', @@ -41,10 +41,10 @@ AND test_source_ranges = 1; /*+ update */ UPDATE google.compute.firewalls SET - data__network = '{{ vpc_link }}', - data__direction = '{{ fw_direction }}', - data__sourceRanges = '{{ fw_source_ranges }}', - data__allowed = '{{ fw_allowed }}' + network = '{{ vpc_link }}', + direction = '{{ fw_direction }}', + sourceRanges = '{{ fw_source_ranges }}', + allowed = '{{ fw_allowed }}' WHERE firewall = '{{ fw_name}}' AND project = '{{ project }}' diff --git a/examples/google/k8s-the-hard-way/resources/forwarding_rule.iql b/examples/google/k8s-the-hard-way/resources/forwarding_rule.iql index 2f25e4e..9039d1d 100644 --- a/examples/google/k8s-the-hard-way/resources/forwarding_rule.iql +++ b/examples/google/k8s-the-hard-way/resources/forwarding_rule.iql @@ -8,11 +8,11 @@ AND forwardingRule = '{{ forwarding_rule_name }}' INSERT INTO google.compute.forwarding_rules( project, region, - data__name, - data__IPAddress, - data__loadBalancingScheme, - data__portRange, - data__target + name, + IPAddress, + loadBalancingScheme, + portRange, + target ) SELECT '{{ project }}', diff --git a/examples/google/k8s-the-hard-way/resources/health_checks.iql b/examples/google/k8s-the-hard-way/resources/health_checks.iql index 7154450..010329d 100644 --- a/examples/google/k8s-the-hard-way/resources/health_checks.iql +++ b/examples/google/k8s-the-hard-way/resources/health_checks.iql @@ -6,15 +6,15 @@ AND httpHealthCheck = '{{ health_check_name }}' /*+ create */ INSERT INTO google.compute.http_health_checks( project, - data__name, - data__checkIntervalSec, - data__description, - data__healthyThreshold, - data__host, - data__port, - data__requestPath, - data__timeoutSec, - data__unhealthyThreshold + name, + checkIntervalSec, + description, + healthyThreshold, + host, + port, + requestPath, + timeoutSec, + unhealthyThreshold ) SELECT '{{ project }}', diff --git a/examples/google/k8s-the-hard-way/resources/instances.iql b/examples/google/k8s-the-hard-way/resources/instances.iql index bf482fa..5a40e80 100644 --- a/examples/google/k8s-the-hard-way/resources/instances.iql +++ b/examples/google/k8s-the-hard-way/resources/instances.iql @@ -16,15 +16,15 @@ INSERT INTO google.compute.instances ( zone, project, - data__name, - data__machineType, - data__canIpForward, - data__deletionProtection, - data__scheduling, - data__networkInterfaces, - data__disks, - data__serviceAccounts, - data__tags + name, + machineType, + canIpForward, + deletionProtection, + scheduling, + networkInterfaces, + disks, + serviceAccounts, + tags ) SELECT '{{ default_zone }}', diff --git a/examples/google/k8s-the-hard-way/resources/network.iql b/examples/google/k8s-the-hard-way/resources/network.iql index c1b39d7..63ac11f 100644 --- a/examples/google/k8s-the-hard-way/resources/network.iql +++ b/examples/google/k8s-the-hard-way/resources/network.iql @@ -7,9 +7,9 @@ AND project = '{{ project }}' INSERT INTO google.compute.networks ( project, - data__name, - data__autoCreateSubnetworks, - data__routingConfig + name, + autoCreateSubnetworks, + routingConfig ) SELECT '{{ project }}', @@ -19,8 +19,8 @@ false, /*+ update */ UPDATE google.compute.networks -SET data__autoCreateSubnetworks = false -AND data__routingConfig = '{"routingMode": "REGIONAL"}' +SET autoCreateSubnetworks = false +AND routingConfig = '{"routingMode": "REGIONAL"}' WHERE network = '{{ vpc_name }}' AND project = '{{ project }}' /*+ statecheck, retries=5, retry_delay=10 */ diff --git a/examples/google/k8s-the-hard-way/resources/public_address.iql b/examples/google/k8s-the-hard-way/resources/public_address.iql index 022db98..bdb7795 100644 --- a/examples/google/k8s-the-hard-way/resources/public_address.iql +++ b/examples/google/k8s-the-hard-way/resources/public_address.iql @@ -9,7 +9,7 @@ INSERT INTO google.compute.addresses ( project, region, - data__name + name ) SELECT '{{ project }}', diff --git a/examples/google/k8s-the-hard-way/resources/routes.iql b/examples/google/k8s-the-hard-way/resources/routes.iql index e40be78..502a77b 100644 --- a/examples/google/k8s-the-hard-way/resources/routes.iql +++ b/examples/google/k8s-the-hard-way/resources/routes.iql @@ -12,11 +12,11 @@ AND name IN ({% for i in range(num_routes | int) %}'{{ route_name_prefix }}-{{ l {% for route in route_data | from_json %} INSERT INTO google.compute.routes( project, - data__destRange, - data__name, - data__network, - data__nextHopIp, - data__priority + destRange, + name, + network, + nextHopIp, + priority ) SELECT '{{ project }}', diff --git a/examples/google/k8s-the-hard-way/resources/subnetwork.iql b/examples/google/k8s-the-hard-way/resources/subnetwork.iql index 7d55eb7..f2d8749 100644 --- a/examples/google/k8s-the-hard-way/resources/subnetwork.iql +++ b/examples/google/k8s-the-hard-way/resources/subnetwork.iql @@ -10,10 +10,10 @@ INSERT INTO google.compute.subnetworks ( project, region, - data__name, - data__network, - data__ipCidrRange, - data__privateIpGoogleAccess + name, + network, + ipCidrRange, + privateIpGoogleAccess ) SELECT '{{ project }}', @@ -26,10 +26,10 @@ true /*+ update */ UPDATE google.compute.subnetworks SET -data__name = '{{ subnet_name }}', -data__network = '{{ vpc_link }}', -data__ipCidrRange = '{{ ip_cidr_range }}', -data__privateIpGoogleAccess = true +name = '{{ subnet_name }}', +network = '{{ vpc_link }}', +ipCidrRange = '{{ ip_cidr_range }}', +privateIpGoogleAccess = true WHERE subnetwork = '{{ subnet_name }}' AND project = '{{ project }}' AND region = '{{ region }}' diff --git a/examples/google/k8s-the-hard-way/resources/target_pool.iql b/examples/google/k8s-the-hard-way/resources/target_pool.iql index 66db671..207c661 100644 --- a/examples/google/k8s-the-hard-way/resources/target_pool.iql +++ b/examples/google/k8s-the-hard-way/resources/target_pool.iql @@ -8,10 +8,10 @@ WHERE project = '{{ project }}' INSERT INTO google.compute.target_pools( project, region, - data__name, - data__healthChecks, - data__instances, - data__sessionAffinity + name, + healthChecks, + instances, + sessionAffinity ) SELECT '{{ project }}', diff --git a/examples/snowflake/entitlements/resources/databases.iql b/examples/snowflake/entitlements/resources/databases.iql index 3f39f52..a763a91 100644 --- a/examples/snowflake/entitlements/resources/databases.iql +++ b/examples/snowflake/entitlements/resources/databases.iql @@ -24,18 +24,18 @@ AND user_task_timeout_ms = {{ user_task_timeout_ms }} /*+ create */ INSERT INTO snowflake.database.databases ( -data__name, -data__kind, -data__comment, -data__data_retention_time_in_days, -data__log_level, -data__max_data_extension_time_in_days, -data__suspend_task_after_num_failures, -data__trace_level, -data__user_task_managed_initial_warehouse_size, -data__serverless_task_min_statement_size, -data__serverless_task_max_statement_size, -data__user_task_timeout_ms, +name, +kind, +comment, +data_retention_time_in_days, +log_level, +max_data_extension_time_in_days, +suspend_task_after_num_failures, +trace_level, +user_task_managed_initial_warehouse_size, +serverless_task_min_statement_size, +serverless_task_max_statement_size, +user_task_timeout_ms, endpoint ) SELECT @@ -71,7 +71,7 @@ serverless_task_max_statement_size = '{{ serverless_task_max_statement_size }}', user_task_timeout_ms = '{{ user_task_timeout_ms }}' WHERE name = '{{ database_name }}' -AND data__name = '{{ database_name }}' +AND name = '{{ database_name }}' AND endpoint = '{{ endpoint }}'; /*+ exports */ diff --git a/examples/snowflake/entitlements/resources/grants.iql b/examples/snowflake/entitlements/resources/grants.iql index 93c65b7..d6beff7 100644 --- a/examples/snowflake/entitlements/resources/grants.iql +++ b/examples/snowflake/entitlements/resources/grants.iql @@ -1,9 +1,9 @@ /*+ createorupdate */ INSERT INTO snowflake.role.grants ( -data__securable, -data__securable_type, -data__grant_option, -data__privileges, +securable, +securable_type, +grant_option, +privileges, name, endpoint ) @@ -19,5 +19,5 @@ SELECT /*+ delete */ DELETE FROM snowflake.role.grants WHERE name = '{{ role_name }}' -AND data__securable_type = '{{ securable_type }}' +AND securable_type = '{{ securable_type }}' AND endpoint = '{{ endpoint }}'; \ No newline at end of file diff --git a/examples/snowflake/entitlements/resources/masking_policies.iql b/examples/snowflake/entitlements/resources/masking_policies.iql index 108d59b..0d20868 100644 --- a/examples/snowflake/entitlements/resources/masking_policies.iql +++ b/examples/snowflake/entitlements/resources/masking_policies.iql @@ -1,9 +1,9 @@ /*+ createorupdate */ INSERT INTO snowflake.sqlapi.statements ( -data__statement, -data__timeout, -data__database, -data__schema, +statement, +timeout, +database, +schema, "User-Agent", endpoint ) diff --git a/examples/snowflake/entitlements/resources/roles.iql b/examples/snowflake/entitlements/resources/roles.iql index cc8e2b0..3f0aff5 100644 --- a/examples/snowflake/entitlements/resources/roles.iql +++ b/examples/snowflake/entitlements/resources/roles.iql @@ -1,7 +1,7 @@ /*+ createorupdate */ INSERT INTO snowflake.role.roles ( -data__name, -data__comment, +name, +comment, endpoint, createMode ) diff --git a/examples/snowflake/entitlements/resources/schemas.iql b/examples/snowflake/entitlements/resources/schemas.iql index 55a72e6..0e0df9a 100644 --- a/examples/snowflake/entitlements/resources/schemas.iql +++ b/examples/snowflake/entitlements/resources/schemas.iql @@ -25,18 +25,18 @@ AND user_task_timeout_ms = {{ user_task_timeout_ms }}; /*+ create */ INSERT INTO snowflake.schema.schemas ( -data__name, -data__kind, -data__managed_access, -data__data_retention_time_in_days, -data__log_level, -data__max_data_extension_time_in_days, -data__suspend_task_after_num_failures, -data__trace_level, -data__user_task_managed_initial_warehouse_size, -data__serverless_task_min_statement_size, -data__serverless_task_max_statement_size, -data__user_task_timeout_ms, +name, +kind, +managed_access, +data_retention_time_in_days, +log_level, +max_data_extension_time_in_days, +suspend_task_after_num_failures, +trace_level, +user_task_managed_initial_warehouse_size, +serverless_task_min_statement_size, +serverless_task_max_statement_size, +user_task_timeout_ms, database_name, endpoint ) @@ -74,7 +74,7 @@ user_task_timeout_ms = {{ user_task_timeout_ms }} WHERE database_name = '{{ database_name }}' AND name = '{{ schema_name }}' -AND data__name = '{{ schema_name }}' +AND name = '{{ schema_name }}' AND endpoint = '{{ endpoint }}'; /*+ exports */ diff --git a/examples/snowflake/entitlements/resources/statements.iql b/examples/snowflake/entitlements/resources/statements.iql index 18782c0..1393813 100644 --- a/examples/snowflake/entitlements/resources/statements.iql +++ b/examples/snowflake/entitlements/resources/statements.iql @@ -1,9 +1,9 @@ /*+ createorupdate */ INSERT INTO snowflake.sqlapi.statements ( -data__statement, -data__timeout, -data__database, -data__schema, +statement, +timeout, +database, +schema, "User-Agent", endpoint ) diff --git a/examples/snowflake/entitlements/resources/tables.iql b/examples/snowflake/entitlements/resources/tables.iql index 5411d68..a4150a3 100644 --- a/examples/snowflake/entitlements/resources/tables.iql +++ b/examples/snowflake/entitlements/resources/tables.iql @@ -18,11 +18,11 @@ AND comment = '{{ comment }}'; /*+ create */ INSERT INTO snowflake.table.tables ( -data__name, -data__kind, -data__columns, -data__constraints, -data__comment, +name, +kind, +columns, +constraints, +comment, database_name, schema_name, endpoint diff --git a/examples/snowflake/entitlements/resources/warehouses.iql b/examples/snowflake/entitlements/resources/warehouses.iql index 70858ec..112f8b1 100644 --- a/examples/snowflake/entitlements/resources/warehouses.iql +++ b/examples/snowflake/entitlements/resources/warehouses.iql @@ -1,13 +1,13 @@ /*+ createorupdate */ INSERT INTO snowflake.warehouse.warehouses ( -data__name, -data__warehouse_type, -data__warehouse_size, -data__scaling_policy, -data__auto_suspend, -data__auto_resume, -data__initially_suspended, -data__comment, +name, +warehouse_type, +warehouse_size, +scaling_policy, +auto_suspend, +auto_resume, +initially_suspended, +comment, createMode, endpoint ) diff --git a/src/utils/download.rs b/src/utils/download.rs index 80dbd61..3322e72 100644 --- a/src/utils/download.rs +++ b/src/utils/download.rs @@ -23,7 +23,7 @@ //! ``` use std::fs::{self, File}; -use std::io::{self, Write}; +use std::io::{self, Read, Write}; use std::path::{Path, PathBuf}; use std::process::Command; @@ -58,8 +58,11 @@ pub fn download_binary() -> Result { // Download the file with progress bar debug!("Downloading from {}", download_url); - let client = Client::new(); - let response = client + let client = Client::builder() + .timeout(std::time::Duration::from_secs(300)) + .build() + .map_err(|e| AppError::CommandFailed(format!("Failed to create HTTP client: {}", e)))?; + let mut response = client .get(&download_url) .send() .map_err(|e| AppError::CommandFailed(format!("Failed to download: {}", e)))?; @@ -73,12 +76,20 @@ pub fn download_binary() -> Result { .progress_chars("#>-")); let mut file = File::create(&archive_path).map_err(AppError::IoError)?; - let mut _downloaded: u64 = 0; - let stream = response - .bytes() - .map_err(|e| AppError::CommandFailed(format!("Failed to read response: {}", e)))?; - - file.write_all(&stream).map_err(AppError::IoError)?; + let mut buffer = [0u8; 8192]; + let mut downloaded: u64 = 0; + loop { + let bytes_read = response + .read(&mut buffer) + .map_err(|e| AppError::CommandFailed(format!("Failed to read response: {}", e)))?; + if bytes_read == 0 { + break; + } + file.write_all(&buffer[..bytes_read]) + .map_err(AppError::IoError)?; + downloaded += bytes_read as u64; + progress_bar.set_position(downloaded); + } progress_bar.finish_with_message("Download complete"); // Extract the file based on platform diff --git a/template-hub/aws/starter/resources/example_vpc.iql.template b/template-hub/aws/starter/resources/example_vpc.iql.template index d4a727c..ee7bf72 100644 --- a/template-hub/aws/starter/resources/example_vpc.iql.template +++ b/template-hub/aws/starter/resources/example_vpc.iql.template @@ -63,5 +63,5 @@ AND json_extract(tags, '$.StackEnv') = '{% raw %}{{ stack_env }}{% endraw %}' /*+ delete */ DELETE FROM aws.ec2.vpcs -WHERE data__Identifier = '{% raw %}{{ vpc_id }}{% endraw %}' +WHERE Identifier = '{% raw %}{{ vpc_id }}{% endraw %}' AND region = '{% raw %}{{ region }}{% endraw %}'; \ No newline at end of file diff --git a/template-hub/azure/starter/resources/example_res_grp.iql.template b/template-hub/azure/starter/resources/example_res_grp.iql.template index bc09859..0f6b08a 100644 --- a/template-hub/azure/starter/resources/example_res_grp.iql.template +++ b/template-hub/azure/starter/resources/example_res_grp.iql.template @@ -11,7 +11,7 @@ AND resourceGroupName = '{% raw %}{{ resource_group_name }}{% endraw %}' INSERT INTO azure.resources.resource_groups( resourceGroupName, subscriptionId, - data__location + location ) SELECT '{% raw %}{{ resource_group_name }}{% endraw %}', diff --git a/template-hub/google/starter/resources/example_vpc.iql.template b/template-hub/google/starter/resources/example_vpc.iql.template index 8ae8338..2336020 100644 --- a/template-hub/google/starter/resources/example_vpc.iql.template +++ b/template-hub/google/starter/resources/example_vpc.iql.template @@ -11,9 +11,9 @@ AND project = '{% raw %}{{ project }}{% endraw %}' INSERT INTO google.compute.networks ( project, - data__name, - data__autoCreateSubnetworks, - data__routingConfig + name, + autoCreateSubnetworks, + routingConfig ) SELECT '{% raw %}{{ project }}{% endraw %}', @@ -23,8 +23,8 @@ false, /*+ update */ UPDATE google.compute.networks -SET data__autoCreateSubnetworks = false -AND data__routingConfig = '{"routingMode": "REGIONAL"}' +SET autoCreateSubnetworks = false +AND routingConfig = '{"routingMode": "REGIONAL"}' WHERE network = '{% raw %}{{ vpc_name }}{% endraw %}' AND project = '{% raw %}{{ project }}{% endraw %}' /*+ statecheck, retries=5, retry_delay=10 */ diff --git a/website/docs/getting-started.md b/website/docs/getting-started.md index f7fe366..9dd0824 100644 --- a/website/docs/getting-started.md +++ b/website/docs/getting-started.md @@ -144,7 +144,7 @@ AND resourceGroupName = '{{ resource_group_name }}' INSERT INTO azure.resources.resource_groups( resourceGroupName, subscriptionId, - data__location + location ) SELECT '{{ resource_group_name }}', diff --git a/website/docs/resource-query-files.md b/website/docs/resource-query-files.md index 13718e2..89799dc 100644 --- a/website/docs/resource-query-files.md +++ b/website/docs/resource-query-files.md @@ -53,9 +53,9 @@ AND project = '{{ project }}' INSERT INTO google.compute.networks ( project, - data__name, - data__autoCreateSubnetworks, - data__routingConfig + name, + autoCreateSubnetworks, + routingConfig ) SELECT '{{ project }}', @@ -74,9 +74,9 @@ INSERT INTO azure.network.virtual_networks( virtualNetworkName, resourceGroupName, subscriptionId, - data__location, - data__properties, - data__tags + location, + properties, + tags ) SELECT '{{ vnet_name }}', @@ -124,12 +124,12 @@ stackql >>show methods in azure.network.virtual_networks; ```sql /*+ createorupdate */ update aws.s3.buckets -set data__PatchDocument = string('{{ { +set PatchDocument = string('{{ { "NotificationConfiguration": transfer_notification_config } | generate_patch_document }}') WHERE region = '{{ region }}' -AND data__Identifier = '{{ transfer_bucket_name }}'; +AND Identifier = '{{ transfer_bucket_name }}'; ``` ### `delete` @@ -352,7 +352,7 @@ INSERT INTO google.compute.addresses ( project, region, - data__name + name ) SELECT '{{ project }}', @@ -454,7 +454,7 @@ SELECT target_key_id as logging_kms_key_id FROM aws.kms.aliases WHERE region = '{{ region }}' -AND data__Identifier = 'alias/{{ stack_name }}/{{ stack_env }}/logging'; +AND Identifier = 'alias/{{ stack_name }}/{{ stack_env }}/logging'; ``` \ No newline at end of file diff --git a/website/docs/template-filters.md b/website/docs/template-filters.md index a8827f4..f3fa810 100644 --- a/website/docs/template-filters.md +++ b/website/docs/template-filters.md @@ -54,12 +54,12 @@ Generates a patch document according to [RFC6902](https://datatracker.ietf.org/d ```sql update aws.s3.buckets -set data__PatchDocument = string('{{ { +set PatchDocument = string('{{ { "NotificationConfiguration": transfer_notification_config } | generate_patch_document }}') WHERE region = '{{ region }}' -AND data__Identifier = '{{ bucket_name }}'; +AND Identifier = '{{ bucket_name }}'; ``` ### `base64_encode` @@ -100,7 +100,7 @@ Escapes a string for use as a SQL string literal by doubling any single quotes. ```sql INSERT INTO snowflake.sqlapi.statements ( -data__statement, +statement, /* other fields... */ ) SELECT @@ -158,7 +158,7 @@ StackQL Deploy injects the following built-in variables automatically — no man The name of the stack as declared in `stackql_manifest.yml`. Available in every template context. ```sql -INSERT INTO google.compute.networks (project, data__name) +INSERT INTO google.compute.networks (project, name) SELECT '{{ project }}', '{{ stack_name }}-{{ stack_env }}-vpc' ``` @@ -172,7 +172,7 @@ The name of the resource currently being processed. Available in every resource ```sql /*+ create */ -INSERT INTO google.logging.sinks (parent, data__name) +INSERT INTO google.logging.sinks (parent, name) SELECT 'projects/{{ project }}', '{{ resource_name }}-sink' ``` @@ -229,7 +229,7 @@ Generates a random UUID (version 4). Useful for creating unique identifiers. ```sql INSERT INTO aws.s3.buckets ( /* fields... */ - data__BucketName, + BucketName, /* other fields... */ ) SELECT diff --git a/website/docs/template-library/aws/vpc-and-ec2-instance.md b/website/docs/template-library/aws/vpc-and-ec2-instance.md index b82b257..c5095f0 100644 --- a/website/docs/template-library/aws/vpc-and-ec2-instance.md +++ b/website/docs/template-library/aws/vpc-and-ec2-instance.md @@ -271,7 +271,7 @@ AND json_extract(tags, '$.StackEnv') = '{{ stack_env }}' /*+ delete */ DELETE FROM aws.ec2.vpcs -WHERE data__Identifier = '{{ vpc_id }}' +WHERE Identifier = '{{ vpc_id }}' AND region = '{{ region }}'; ``` @@ -343,7 +343,7 @@ WHERE cidr_block = '{{ subnet_cidr_block }}'; /*+ delete */ DELETE FROM aws.ec2.subnets -WHERE data__Identifier = '{{ subnet_id }}' +WHERE Identifier = '{{ subnet_id }}' AND region = '{{ region }}'; ``` diff --git a/website/docs/template-library/azure/simple-vnet-and-vm.md b/website/docs/template-library/azure/simple-vnet-and-vm.md index c79e09b..c1fb899 100644 --- a/website/docs/template-library/azure/simple-vnet-and-vm.md +++ b/website/docs/template-library/azure/simple-vnet-and-vm.md @@ -218,9 +218,9 @@ INSERT INTO azure.network.virtual_networks( virtualNetworkName, resourceGroupName, subscriptionId, - data__location, - data__properties, - data__tags + location, + properties, + tags ) SELECT '{{ vnet_name }}', @@ -258,7 +258,7 @@ INSERT INTO azure.network.subnets( virtualNetworkName, resourceGroupName, subscriptionId, - data__properties + properties ) SELECT '{{ subnet_name }}', diff --git a/website/docs/template-library/google/k8s-the-hard-way.md b/website/docs/template-library/google/k8s-the-hard-way.md index 979af5e..3bf697e 100644 --- a/website/docs/template-library/google/k8s-the-hard-way.md +++ b/website/docs/template-library/google/k8s-the-hard-way.md @@ -316,9 +316,9 @@ AND project = '{{ project }}' INSERT INTO google.compute.networks ( project, - data__name, - data__autoCreateSubnetworks, - data__routingConfig + name, + autoCreateSubnetworks, + routingConfig ) SELECT '{{ project }}', @@ -328,8 +328,8 @@ false, /*+ update */ UPDATE google.compute.networks -SET data__autoCreateSubnetworks = false -AND data__routingConfig = '{"routingMode": "REGIONAL"}' +SET autoCreateSubnetworks = false +AND routingConfig = '{"routingMode": "REGIONAL"}' WHERE network = '{{ vpc_name }}' AND project = '{{ project }}' /*+ statecheck, retries=5, retry_delay=10 */ @@ -365,11 +365,11 @@ AND name = '{{ fw_name }}' INSERT INTO google.compute.firewalls ( project, - data__name, - data__network, - data__direction, - data__sourceRanges, - data__allowed + name, + network, + direction, + sourceRanges, + allowed ) SELECT '{{ project }}', @@ -399,10 +399,10 @@ AND test_source_ranges = 1; /*+ update */ UPDATE google.compute.firewalls SET - data__network = '{{ vpc_link }}', - data__direction = '{{ fw_direction }}', - data__sourceRanges = '{{ fw_source_ranges }}', - data__allowed = '{{ fw_allowed }}' + network = '{{ vpc_link }}', + direction = '{{ fw_direction }}', + sourceRanges = '{{ fw_source_ranges }}', + allowed = '{{ fw_allowed }}' WHERE firewall = '{{ fw_name}}' AND project = '{{ project }}'