Sync monorepo state at "Prepare ucconfig for new release" (#34)

Syncing from
userclouds/userclouds@9d79169

e2e-test/lots-of-resources-modified.yaml

@@ -107,7 +107,7 @@ resources:
         access_policy: '@UC_MANIFEST_ID("access_policy_TestPolicy").id'
         columns:
             - column: '@UC_MANIFEST_ID("userstore_column_name").id'
-              validator: '@UC_SYSTEM_OBJECT("transformer", "PassthroughUnchangedData")'
+              normalizer: '@UC_SYSTEM_OBJECT("transformer", "PassthroughUnchangedData")'
         description: Hello world!
         name: BasicMutator
         selector_config:
@@ -131,7 +131,7 @@ resources:
               template_parameters: '{}'
         description: Simple test policy that allows all access
         name: TestPolicy
-        policy_type: compositeunion
+        policy_type: composite_or
         tag_ids: []
     - uc_terraform_type: access_policy_template
       manifest_id: access_policy_template_BasicTestTemplate

e2e-test/lots-of-resources.yaml

@@ -129,9 +129,9 @@ resources:
         access_policy: '@UC_MANIFEST_ID("access_policy_TestPolicy").id'
         columns:
             - column: '@UC_MANIFEST_ID("userstore_column_name").id'
-              validator: '@UC_SYSTEM_OBJECT("transformer", "PassthroughUnchangedData")'
+              normalizer: '@UC_SYSTEM_OBJECT("transformer", "PassthroughUnchangedData")'
             - column: '@UC_MANIFEST_ID("userstore_column_testcol1").id'
-              validator: '@UC_SYSTEM_OBJECT("transformer", "PassthroughUnchangedData")'
+              normalizer: '@UC_SYSTEM_OBJECT("transformer", "PassthroughUnchangedData")'
         description: Hello world
         name: BasicMutator
         selector_config:
@@ -156,7 +156,7 @@ resources:
             - template: '@UC_SYSTEM_OBJECT("access_policy_template", "AllowAll")'
               template_parameters: '{}'
         name: TestPolicy
-        policy_type: compositeunion
+        policy_type: composite_or
         tag_ids: []
     - uc_terraform_type: access_policy_template
       manifest_id: access_policy_template_BasicTestTemplate

e2e-test/test.py

@@ -18,52 +18,59 @@
 TEST_DIR = os.path.dirname(__file__)
 
 # https://stackoverflow.com/a/14693789
-ANSI_ESCAPE_REGEXP = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
+ANSI_ESCAPE_REGEXP = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
 RESOURCE_DEFINITION_REGEXP = re.compile(r'TerraformTypeSuffix:\s+"([a-z0-9_]+)"')
 
+
 @contextmanager
 def log_step(message):
     if os.getenv("GITHUB_ACTIONS"):
-        print(f'::group::{message}')
+        print(f"::group::{message}")
     else:
-        print(f'\033[93m{message}\033[00m')
+        print(f"\033[93m{message}\033[00m")
     try:
         yield
     finally:
         if os.getenv("GITHUB_ACTIONS"):
-            print('::endgroup::')
+            print("::endgroup::")
+
 
 class ChildProcessError(Exception):
     def __init__(self, output):
         self.output = output
 
+
 def run_capturing_output(cmd):
-    output = ''
+    output = ""
     process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     for line in process.stdout:
-        decoded = line.decode('utf-8')
+        decoded = line.decode("utf-8")
         # Keep color codes in the output to the terminal, but strip them for the
         # captured output string so that we can search that string without color
         # codes messing up the match
-        print(decoded.rstrip('\n'))
-        output += ANSI_ESCAPE_REGEXP.sub('', decoded)
+        print(decoded.rstrip("\n"))
+        output += ANSI_ESCAPE_REGEXP.sub("", decoded)
     if process.wait() != 0:
         raise ChildProcessError(output)
     return output
 
+
 def validate_manifest(manifest_name):
     with open(os.path.join(TEST_DIR, manifest_name)) as f:
-        if manifest_name.endswith('.json'):
+        if manifest_name.endswith(".json"):
             manifest = json.load(f)
-        elif manifest_name.endswith('.yaml'):
+        elif manifest_name.endswith(".yaml"):
             manifest = yaml.safe_load(f)
         else:
-            raise Exception(f'Unknown manifest file extension: {manifest_name}')
-    for resource in manifest['resources']:
-        if '<<TARGET_FQTN>>' not in resource['resource_uuids']:
-            raise Exception(f'<<TARGET_FQTN>> not found in resource_uuids for resource {resource["manifest_id"]}. '
-                            + 'This is required so that the tests work against arbitrary tenants. '
-                            + 'Please replace your tenant FQTN with <<TARGET_FQTN>>.')
+            raise Exception(f"Unknown manifest file extension: {manifest_name}")
+    for resource in manifest["resources"]:
+        if "<<TARGET_FQTN>>" not in resource["resource_uuids"]:
+            raise Exception(
+                f'<<TARGET_FQTN>> not found in resource_uuids for resource {resource["manifest_id"]}. '
+                + "This is required so that the tests work against arbitrary tenants. "
+                + "Please replace your tenant FQTN with <<TARGET_FQTN>>."
+            )
+
 
 @contextmanager
 def manifest_with_real_fqtn(manifest_name):
@@ -76,112 +83,166 @@ def manifest_with_real_fqtn(manifest_name):
     <<TARGET_FQTN>> in the e2e test manifests, and replace them with the real
     FQTN when we use them.
     """
-    tenant_url = os.getenv('USERCLOUDS_TENANT_URL')
+    tenant_url = os.getenv("USERCLOUDS_TENANT_URL")
     if not tenant_url:
-        raise Exception('USERCLOUDS_TENANT_URL is required')
-    fqtn = urlparse(tenant_url).hostname.split('.')[0]
+        raise Exception("USERCLOUDS_TENANT_URL is required")
+    fqtn = urlparse(tenant_url).hostname.split(".")[0]
 
     # Note: we want to make a temporary file in the same directory as the
     # original manifest, so that relative references to e.g. transformer
     # definitions within that directory still work.
-    target_path = os.path.join(TEST_DIR, manifest_name + '.substituted-tmp.' + manifest_name.split('.')[-1])
-    with open(os.path.join(TEST_DIR, target_path), 'w') as rewritten:
+    target_path = os.path.join(
+        TEST_DIR, manifest_name + ".substituted-tmp." + manifest_name.split(".")[-1]
+    )
+    with open(os.path.join(TEST_DIR, target_path), "w") as rewritten:
         with open(os.path.join(TEST_DIR, manifest_name)) as original:
             for line in original:
-                rewritten.write(line.replace('<<TARGET_FQTN>>', fqtn))
+                rewritten.write(line.replace("<<TARGET_FQTN>>", fqtn))
     try:
         yield target_path
     finally:
         os.remove(target_path)
 
+
 def apply_manifest(manifest_name, extra_args):
     with manifest_with_real_fqtn(manifest_name) as manifest_path:
-        return run_capturing_output(['ucconfig', 'apply', manifest_path, '--auto-approve'] + extra_args)
+        return run_capturing_output(
+            ["ucconfig", "apply", manifest_path, "--auto-approve"] + extra_args
+        )
+
 
 def assert_no_changes(manifest_name, extra_args):
     with manifest_with_real_fqtn(manifest_name) as manifest_path:
-        out = run_capturing_output(['ucconfig', 'apply', manifest_path, '--dry-run'] + extra_args)
-        if 'No changes. Your infrastructure matches the configuration.' not in out:
-            raise Exception('Got an unexpected diff!')
+        out = run_capturing_output(
+            ["ucconfig", "apply", manifest_path, "--dry-run"] + extra_args
+        )
+        if "No changes. Your infrastructure matches the configuration." not in out:
+            raise Exception("Got an unexpected diff!")
         return out
 
+
 def assert_genmanifest_matches(manifest_name):
     with manifest_with_real_fqtn(manifest_name) as base_manifest_path:
         with tempfile.TemporaryDirectory() as tmpdirname:
-            run_capturing_output(['ucconfig', 'gen-manifest', os.path.join(tmpdirname, manifest_name)])
+            run_capturing_output(
+                ["ucconfig", "gen-manifest", os.path.join(tmpdirname, manifest_name)]
+            )
             diff_found = False
 
             # Compare manifest file
-            if not filecmp.cmp(base_manifest_path, os.path.join(tmpdirname, manifest_name)):
-                print('Running gen-manifest modified the manifest:')
-                subprocess.run(['diff', '-u', base_manifest_path, os.path.join(tmpdirname, manifest_name)])
+            if not filecmp.cmp(
+                base_manifest_path, os.path.join(tmpdirname, manifest_name)
+            ):
+                print("Running gen-manifest modified the manifest:")
+                subprocess.run(
+                    [
+                        "diff",
+                        "-u",
+                        base_manifest_path,
+                        os.path.join(tmpdirname, manifest_name),
+                    ]
+                )
                 diff_found = True
 
             # Compare values files
-            values_dirname = Path(manifest_name).stem + '_values'
+            values_dirname = Path(manifest_name).stem + "_values"
             existing_values_dir = os.path.join(TEST_DIR, values_dirname)
             generated_values_dir = os.path.join(tmpdirname, values_dirname)
             values_cmp = filecmp.dircmp(existing_values_dir, generated_values_dir)
             for f in values_cmp.left_only:
-                print(f'Running genmanifest deleted {values_dirname}/{f}')
+                print(f"Running genmanifest deleted {values_dirname}/{f}")
                 diff_found = True
             for f in values_cmp.right_only:
-                print(f'Running genmanifest created {values_dirname}/{f}')
+                print(f"Running genmanifest created {values_dirname}/{f}")
                 diff_found = True
             for f in values_cmp.diff_files:
-                print(f'Running genmanifest modified {values_dirname}/{f}:')
-                subprocess.run(['diff', '-u', os.path.join(existing_values_dir, f), os.path.join(generated_values_dir, f)])
-                print('') # add blank line after diff output as separation
+                print(f"Running genmanifest modified {values_dirname}/{f}:")
+                subprocess.run(
+                    [
+                        "diff",
+                        "-u",
+                        os.path.join(existing_values_dir, f),
+                        os.path.join(generated_values_dir, f),
+                    ]
+                )
+                print("")  # add blank line after diff output as separation
                 diff_found = True
 
             if diff_found:
-                raise Exception('genmanifest produced a differing manifest and/or external value files')
+                raise Exception(
+                    "genmanifest produced a differing manifest and/or external value files"
+                )
+
 
 def list_resource_types():
-    with open(os.path.join(TEST_DIR, '../internal/resourcetypes/types.go')) as f:
+    with open(os.path.join(TEST_DIR, "../internal/resourcetypes/types.go")) as f:
         return RESOURCE_DEFINITION_REGEXP.findall(f.read())
 
+
 def main():
     # Pass extra arguments onto ucconfig. (In the future maybe we'll want to
     # argparse here, but we have no other arguments at this time)
     ucconfig_apply_args = sys.argv[1:]
 
     # A previous failed run could have left unwanted resources. Start by
     # deleting all non-system resources.
-    validate_manifest('empty.yaml')
+    validate_manifest("empty.yaml")
     with log_step("Applying empty.yaml to get to baseline state..."):
-        apply_manifest('empty.yaml', ucconfig_apply_args)
+        apply_manifest("empty.yaml", ucconfig_apply_args)
     with log_step("Applying empty.yaml again should not change anything..."):
-        assert_no_changes('empty.yaml', ucconfig_apply_args)
+        assert_no_changes("empty.yaml", ucconfig_apply_args)
 
     # Apply config with a bunch of resources
     with log_step("Applying lots-of-resources.yaml to test resource creation..."):
-        validate_manifest('lots-of-resources.yaml')
-        output = apply_manifest('lots-of-resources.yaml', ucconfig_apply_args)
+        validate_manifest("lots-of-resources.yaml")
+        output = apply_manifest("lots-of-resources.yaml", ucconfig_apply_args)
         for tf_type_suffix in list_resource_types():
-            if not re.search(f'userclouds_{tf_type_suffix}' + r'\.[a-zA-Z0-9_-]+ will be created', output):
-                raise Exception(f'Did not see resource type {tf_type_suffix} created in the output. Please add test coverage')
-    with log_step("Applying lots-of-resources.yaml again should not change anything..."):
-        assert_no_changes('lots-of-resources.yaml', ucconfig_apply_args)
-    with log_step("Regenerating lots-of-resources.yaml should generate the same manifest..."):
-        assert_genmanifest_matches('lots-of-resources.yaml')
-
-    with log_step("Applying lots-of-resources-modified.yaml to test resource modification..."):
-        validate_manifest('lots-of-resources-modified.yaml')
-        output = apply_manifest('lots-of-resources-modified.yaml', ucconfig_apply_args)
+            if not re.search(
+                f"userclouds_{tf_type_suffix}" + r"\.[a-zA-Z0-9_-]+ will be created",
+                output,
+            ):
+                raise Exception(
+                    f"Did not see resource type {tf_type_suffix} created in the output. Please add test coverage"
+                )
+    with log_step(
+        "Applying lots-of-resources.yaml again should not change anything..."
+    ):
+        assert_no_changes("lots-of-resources.yaml", ucconfig_apply_args)
+
+    with log_step(
+        "Regenerating lots-of-resources.yaml should generate the same manifest..."
+    ):
+        assert_genmanifest_matches("lots-of-resources.yaml")
+
+    with log_step(
+        "Applying lots-of-resources-modified.yaml to test resource modification..."
+    ):
+        validate_manifest("lots-of-resources-modified.yaml")
+        output = apply_manifest("lots-of-resources-modified.yaml", ucconfig_apply_args)
         for tf_type_suffix in list_resource_types():
-            if not re.search(f'userclouds_{tf_type_suffix}' + r'\.[a-zA-Z0-9_-]+ (?:will be updated|must be replaced)', output):
-                raise Exception(f'Did not see resource type {tf_type_suffix} updated in the output. Please add test coverage')
-    with log_step("Applying lots-of-resources-modified.yaml again should not change anything..."):
-        assert_no_changes('lots-of-resources-modified.yaml', ucconfig_apply_args)
-    with log_step("Regenerating lots-of-resources-modified.yaml should generate the same manifest..."):
-        assert_genmanifest_matches('lots-of-resources-modified.yaml')
+            if not re.search(
+                f"userclouds_{tf_type_suffix}"
+                + r"\.[a-zA-Z0-9_-]+ (?:will be updated|must be replaced)",
+                output,
+            ):
+                raise Exception(
+                    f"Did not see resource type {tf_type_suffix} updated in the output. Please add test coverage"
+                )
+    with log_step(
+        "Applying lots-of-resources-modified.yaml again should not change anything..."
+    ):
+        assert_no_changes("lots-of-resources-modified.yaml", ucconfig_apply_args)
+    with log_step(
+        "Regenerating lots-of-resources-modified.yaml should generate the same manifest..."
+    ):
+        assert_genmanifest_matches("lots-of-resources-modified.yaml")
 
     # Apply empty config
     with log_step("Applying empty.yaml to test resource deletion..."):
-        apply_manifest('empty.yaml', ucconfig_apply_args)
+        apply_manifest("empty.yaml", ucconfig_apply_args)
     with log_step("Applying empty.yaml again should not change anything..."):
-        assert_no_changes('empty.yaml', ucconfig_apply_args)
+        assert_no_changes("empty.yaml", ucconfig_apply_args)
+
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

go.mod

@@ -9,14 +9,15 @@ require (
 	github.com/zclconf/go-cty v1.14.0
 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa
 	gopkg.in/yaml.v3 v3.0.1
-	userclouds.com v0.7.7
+	userclouds.com v1.0.0
 )
 
 require (
 	github.com/agext/levenshtein v1.2.1 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/kr/pretty v0.3.1 // indirect

go.sum

@@ -16,6 +16,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a h1:v6zMvHuY9yue4+QkG/HQ/W67wvtQmWJ4SDo9aK/GIno=
+github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a/go.mod h1:I79BieaU4fxrw4LMXby6q5OS9XnoR9UIKLOzDFjUmuw=
 github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
 github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
 github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
@@ -42,6 +44,7 @@ github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZX
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/redis/go-redis/v9 v9.3.1 h1:KqdY8U+3X6z+iACvumCNxnoluToB+9Me+TvyFa21Mds=
 github.com/redis/go-redis/v9 v9.3.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
@@ -52,6 +55,7 @@ github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
 github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/zclconf/go-cty v1.14.0 h1:/Xrd39K7DXbHzlisFP9c4pHao4yyf+/Ug9LEz+Y/yhc=
 github.com/zclconf/go-cty v1.14.0/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ=
@@ -64,5 +68,5 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-userclouds.com v0.7.7 h1:wJ0rJyNJy2Ag39g8cjGgANGSBRlWRuDoTRv6UCDMBy8=
-userclouds.com v0.7.7/go.mod h1:8iP9wmGFuP9bxUiAIjWxwWdlWxJx94gQp6ldeL3l9s4=
+userclouds.com v1.0.0 h1:a7mIfZ0wX1twSAJGJM6JFi2rjFQxetRKCzZ/wkVOzTA=
+userclouds.com v1.0.0/go.mod h1:LhonLJTDVJfdbTGocj/D9BEDV1j0TDWMmnNxiNjST2E=