V3 alias refactoring (#3872)

Try to get the right information into the right places in order to
implement robust auto-aliasing to address
#3848

.gitignore

@@ -20,6 +20,7 @@ provider/cmd/**/metadata-compact.json
 provider/cmd/**/*.gz
 provider/**/testdata/rapid
 provider/pkg/versionLookup/v*-lock.json
+provider/pkg/versionLookup/default-versions.yaml
 **/version.txt
 **/nuget
 **/dist

CONTRIBUTING.md

@@ -75,13 +75,14 @@ Updates to the version selection is done conservatively within the same major ve
 
 ### Configuration Files
 
-Version configuration files are prefixed with the major version they're for (e.g. `v2-`) and live within the `versions/` folder. There's three steps to the versioning process:
+Version configuration files are prefixed with the major version they're for (e.g. `v2-`) and live within the `versions/` folder.
 
 1. Config: (e.g. [`v2-config.json`](./versions/v2-config.json)) is a hand-edited file to control how the Spec is updated.
 2. Spec: (e.g. [`v2-spec.yaml`](./versions/v2-spec.yaml)) is automatically appended to but can be manually edited.
-3. Lock: (e.g. [`v2-lock.json`](./versions/v2-lock.json)) is generated from the Spec and is the exact selection of resource versions to use.
+3. Removed resources/invokes: (e.g. [`v2-removed-resources.json`](./versions/v2-removed-resources.json))
+4. Removed versions: (e.g. [`v2-removed.json`](./versions/v2-removed.json))
 
-The default version selection is calculated as a part of the `make schema` target.
+The default versions file (e.g. [`v2.yaml`](./versions/v2.yaml)) is calculated as a part of the `make schema` target and shouldn't be modified directly.
 
 ### Config
 
@@ -100,7 +101,7 @@ A spec file consists of service names, each specifying:
 
 Both `tracking` and `additions` can be specified together but if the set of resources overlaps (the same resource exists) in both the tracking version and the additions, then the versioning program will fail with an error.
 
-From the Spec file, we generate the Lock JSON file (e.g. [`v2-lock.json`](./versions/v2-lock.json)) which contain the fully expanded set of default resources. These are the files used at the point of generating the schema.
+From the Spec file, we calculate the default versions file (e.g. [`v2.yaml`](./versions/v2.yaml)) which contain the fully expanded set of resource versions for each module. These are the files used at the point of generating the schema.
 
 ### Creating a new major version default selection
 
@@ -116,9 +117,9 @@ Supporting every version of every API causes the SDK size to be very large. One
 
 Ideally, at the point a new incompatible version of the type is added, it should then be created with a unique, stable name. Alternatively, we could create a union of the two possible types
 
-### Default Version Locks
+### Default Versions File
 
-The default version is calculated and written to a 'lock' file which list every resource (or invoke) at a specific API version. This file is read in during the schema and SDK generation. Currently there is a `v1-lock.json` and `v2-lock.json` lock file. These lock files should not be edited directly, but should be calculated by the versioner tool using a specific algorithm.
+The default version is calculated and written to a YAML file which list every resource (or invoke) at a specific API version. This file is read in during the schema and SDK generation. Currently there is a `v1.json` and `v2.json` file. These files should not be edited directly, but should be calculated by the versioner tool using a specific algorithm.
 
 ### Additional Reports
 

Makefile

@@ -231,12 +231,12 @@ bin/$(CODEGEN): .make/prebuild .make/provider_mod_download provider/cmd/$(CODEGE
 
 # Writes schema-full.json and metadata-compact.json to bin/
 # Also re-calculates files in versions/ at same time
-bin/schema-full.json bin/metadata-compact.json &: bin/$(CODEGEN) $(SPECS) versions/az-provider-list.json versions/v${PREVIOUS_MAJOR_VERSION}-lock.json versions/v${MAJOR_VERSION}-config.yaml versions/v${MAJOR_VERSION}-spec.yaml versions/v${MAJOR_VERSION}-removed.json versions/v${MAJOR_VERSION}-removed-resources.json versions/v${NEXT_MAJOR_VERSION}-removed-resources.json
+bin/schema-full.json bin/metadata-compact.json &: bin/$(CODEGEN) $(SPECS) versions/az-provider-list.json versions/v${PREVIOUS_MAJOR_VERSION}.yaml versions/v${MAJOR_VERSION}-config.yaml versions/v${MAJOR_VERSION}-spec.yaml versions/v${MAJOR_VERSION}-removed.json versions/v${MAJOR_VERSION}-removed-resources.json versions/v${NEXT_MAJOR_VERSION}-removed-resources.json
 	bin/$(CODEGEN) schema $(PROVIDER_VERSION)
 
 # Docs schema - treat as phony becasuse it's committed so we always need to rebuild it.
 .PHONY: provider/cmd/pulumi-resource-azure-native/schema.json
-provider/cmd/pulumi-resource-azure-native/schema.json: bin/$(CODEGEN) $(SPECS) versions/v${PREVIOUS_MAJOR_VERSION}-lock.json versions/v${MAJOR_VERSION}-config.yaml versions/v${MAJOR_VERSION}-removed-resources.json
+provider/cmd/pulumi-resource-azure-native/schema.json: bin/$(CODEGEN) $(SPECS) versions/v${PREVIOUS_MAJOR_VERSION}.yaml versions/v${MAJOR_VERSION}-config.yaml versions/v${MAJOR_VERSION}-removed-resources.json
 	bin/$(CODEGEN) docs $(PROVIDER_VERSION)
 
 bin/$(LOCAL_PROVIDER_FILENAME): .make/prebuild .make/provider_mod_download provider/cmd/$(PROVIDER)/*.go .make/provider_prebuild $(PROVIDER_PKG)
@@ -305,13 +305,13 @@ dist/pulumi-azure-native_$(PROVIDER_VERSION)_checksums.txt: dist/$(PROVIDER)-v$(
 	cd provider && go mod download
 	@touch $@
 
-.make/provider_prebuild: .make/prebuild bin/schema-full.json bin/metadata-compact.json versions/v${MAJOR_VERSION}-lock.json
+.make/provider_prebuild: .make/prebuild bin/schema-full.json bin/metadata-compact.json versions/v${MAJOR_VERSION}.yaml
 	cp bin/schema-full.json provider/cmd/$(PROVIDER)
 	cp bin/metadata-compact.json provider/cmd/$(PROVIDER)
 	@touch $@
 
 .make/prebuild: .pulumi/bin/pulumi
-	cp -v versions/v${MAJOR_VERSION}-lock.json provider/pkg/versionLookup/version-lock.json
+	cp -v versions/v${MAJOR_VERSION}.yaml provider/pkg/versionLookup/default-versions.yaml
 	@touch $@
 
 define FAKE_MODULE

provider/cmd/pulumi-gen-azure-native/main.go

@@ -147,7 +147,7 @@ func main() {
 		if err != nil {
 			panic(err)
 		}
-		squeezedInvokes := versioning.FindRemovedInvokesFromResources(buildSchemaResult.Providers, squeezedResources)
+		squeezedInvokes := versioning.FindRemovedInvokesFromResources(buildSchemaResult.Modules, squeezedResources)
 		majorVersion := strings.Split(version, ".")[0]
 		err = gen.EmitFile(path.Join("versions", fmt.Sprintf("v%s-removed-resources.json", majorVersion)), squeezedResources)
 		if err != nil {

provider/pkg/gen/gen_aliases_test.go

@@ -25,13 +25,13 @@ func TestAliasesGen(t *testing.T) {
 	rootDir := path.Join(wd, "test-data", "aliases")
 
 	// azure-rest-api-specs/specification/common-types/resource-management/v1
-	providers, _, err := openapi.ReadAzureProviders(path.Join(rootDir, "azure-rest-api-specs"), "Aadiam", "2020-03-01")
+	modules, _, err := openapi.ReadAzureModules(path.Join(rootDir, "azure-rest-api-specs"), "Aadiam", "2020-03-01")
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	t.Run("v2", func(t *testing.T) {
-		generationResult, err := PulumiSchema(rootDir, providers, versioningStub{}, semver.MustParse("2.0.0"))
+		generationResult, err := PulumiSchema(rootDir, modules, versioningStub{}, semver.MustParse("2.0.0"))
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -41,7 +41,7 @@ func TestAliasesGen(t *testing.T) {
 	})
 
 	t.Run("v3", func(t *testing.T) {
-		generationResult, err := PulumiSchema(rootDir, providers, versioningStub{}, semver.MustParse("3.0.0"))
+		generationResult, err := PulumiSchema(rootDir, modules, versioningStub{}, semver.MustParse("3.0.0"))
 		if err != nil {
 			t.Fatal(err)
 		}

provider/pkg/gen/gen_dashboard_test.go

@@ -25,16 +25,16 @@ func TestPortalDashboardGen(t *testing.T) {
 	// rm -rf provider/pkg/gen/test-data/dashboard/azure-rest-api-specs/specification/portal/resource-manager/Microsoft.Portal/preview/2020-09-01-preview/examples/
 	rootDir := path.Join(wd, "test-data", "dashboard")
 
-	providers, _, err := openapi.ReadAzureProviders(path.Join(rootDir, "azure-rest-api-specs"), "Portal", "2020-09-01-preview")
+	modules, _, err := openapi.ReadAzureModules(path.Join(rootDir, "azure-rest-api-specs"), "Portal", "2020-09-01-preview")
 	if err != nil {
 		t.Fatal(err)
 	}
-	providers = openapi.ApplyProvidersTransformations(providers, openapi.DefaultVersionLock{
+	modules = openapi.ApplyTransformations(modules, openapi.DefaultVersions{
 		"Portal": {
 			"Dashboard": "2020-09-01-preview",
 		},
-	}, openapi.DefaultVersionLock{}, nil, nil)
-	generationResult, err := PulumiSchema(rootDir, providers, versioningStub{}, semver.MustParse("3.0.0"))
+	}, openapi.DefaultVersions{}, nil, nil)
+	generationResult, err := PulumiSchema(rootDir, modules, versioningStub{}, semver.MustParse("3.0.0"))
 	if err != nil {
 		t.Fatal(err)
 	}

provider/pkg/gen/gen_vnet_test.go

@@ -22,11 +22,11 @@ func TestVnetGen(t *testing.T) {
 	// 3. rm -rf provider/pkg/gen/test-data/vnet/azure-rest-api-specs/specification/network/resource-manager/Microsoft.Network/stable/2023-02-01/examples/
 	rootDir := path.Join(wd, "test-data", "vnet")
 
-	providers, _, err := openapi.ReadAzureProviders(path.Join(rootDir, "azure-rest-api-specs"), "Network", "2023-02-01")
+	modules, _, err := openapi.ReadAzureModules(path.Join(rootDir, "azure-rest-api-specs"), "Network", "2023-02-01")
 	if err != nil {
 		t.Fatal(err)
 	}
-	generationResult, err := PulumiSchema(rootDir, providers, versioningStub{}, semver.MustParse("2.0.0"))
+	generationResult, err := PulumiSchema(rootDir, modules, versioningStub{}, semver.MustParse("2.0.0"))
 	if err != nil {
 		t.Fatal(err)
 	}

provider/pkg/gen/properties.go

@@ -133,7 +133,7 @@ func (m *moduleGenerator) genProperties(resolvedSchema *openapi.Schema, variants
 
 		// See #3556 and https://github.com/Azure/azure-rest-api-specs/issues/30443
 		// It's ok if a new API version is fixed, this is a no-op then.
-		if m.resourceName == "CIAMTenant" && name == "tier" && strings.HasPrefix(m.module, "azureactivedirectory") {
+		if m.resourceName == "CIAMTenant" && name == "tier" && strings.HasPrefix(string(m.module), "azureactivedirectory") {
 			flatten = false
 		}
 
@@ -323,7 +323,7 @@ func (m *moduleGenerator) genProperty(name string, schema *spec.Schema, context
 
 	// Special case for KeyVault access policies - they are a resource that can be defined inline in Vaults or
 	// stand-alone. The logic above cannot detect that because they are defined as a custom resource. #594
-	if m.resourceName == "Vault" && m.prov == "KeyVault" && name == "accessPolicies" {
+	if m.resourceName == "Vault" && m.moduleName == "KeyVault" && name == "accessPolicies" {
 		maintainSubResourceIfUnset = true
 	}
 
@@ -513,18 +513,18 @@ func (m *moduleGenerator) forceNew(schema *openapi.Schema, propertyName string,
 	if hasMutabilityInfo && forcesRecreate {
 		if isType {
 			m.forceNewTypes = append(m.forceNewTypes, ForceNewType{
-				Module:        m.module,
-				Provider:      m.prov,
-				ResourceName:  m.resourceName,
-				ReferenceName: schema.ReferenceContext.ReferenceName,
-				Property:      propertyName,
+				VersionedModule: m.module,
+				Module:          m.moduleName,
+				ResourceName:    m.resourceName,
+				ReferenceName:   schema.ReferenceContext.ReferenceName,
+				Property:        propertyName,
 			})
 			return forceNewSetOnReferencedType
 		}
 		return forceNew
 	}
 
-	if resourceMap, ok := forceNewMap[m.prov]; ok {
+	if resourceMap, ok := forceNewMap[m.moduleName]; ok {
 		if properties, ok := resourceMap[m.resourceName]; ok {
 			if properties.Has(propertyName) {
 				return forceNew

provider/pkg/gen/properties_test.go

@@ -73,7 +73,7 @@ func TestPropChangeForcesRecreate(t *testing.T) {
 
 func TestForceNew(t *testing.T) {
 	m := moduleGenerator{
-		prov: "foo",
+		moduleName: "foo",
 	}
 
 	t.Run("forceNew", func(t *testing.T) {
@@ -105,7 +105,7 @@ func TestForceNew(t *testing.T) {
 
 func TestNonObjectInvokeResponses(t *testing.T) {
 	m := moduleGenerator{
-		prov: "foo",
+		moduleName: "foo",
 	}
 
 	resolvedSchema := &openapi.Schema{

provider/pkg/gen/propertyDefaults.go

@@ -7,8 +7,8 @@ import "strings"
 // the omission as "no change" instead of removing the value.
 // propertyDefaults returns a map of property names to default values for the given module
 // and resource, for recording in metadata for later use.
-func propertyDefaults(module, resourceName string) map[string]interface{} {
-	if resourceName == "StorageAccount" && (module == "storage" || strings.HasPrefix(module, "storage/")) {
+func propertyDefaults(module TokenModule, resourceName string) map[string]interface{} {
+	if resourceName == "StorageAccount" && (module == "storage" || strings.HasPrefix(string(module), "storage/")) {
 		return map[string]interface{}{
 			"networkRuleSet": map[string]interface{}{
 				"bypass":              "AzureServices",

provider/pkg/gen/replacement.go

@@ -2,11 +2,14 @@
 
 package gen
 
-import "github.com/pulumi/pulumi/pkg/v3/codegen"
+import (
+	"github.com/pulumi/pulumi-azure-native/v2/provider/pkg/openapi"
+	"github.com/pulumi/pulumi/pkg/v3/codegen"
+)
 
-// forceNewMap is a map of Azure Resource Provider -> Resource Name -> Properties that cause replacements.
+// forceNewMap is a map of Module Name -> Resource Name -> Properties that cause replacements.
 // API Versions are currently ignored.
-var forceNewMap = map[string]map[string]codegen.StringSet{
+var forceNewMap = map[openapi.ModuleName]map[string]codegen.StringSet{
 	"Authorization": {
 		"RoleAssignment": codegen.NewStringSet("principalId", "scope"),
 	},