feat: add support for overlays

.github/workflows/sdk-generation.yaml

@@ -33,6 +33,21 @@ on:
             - https://example.com/openapi2.json
         required: false
         type: string
+      overlay_docs:
+        description: |-
+          A yaml string containing a list of overlay documents to use, if multiple documents are provided they will be applied to the OpenAPI document in the order provided.
+
+          If the document lives within the repo a relative path can be provided, if the document is hosted publicly a URL can be provided.
+
+          If the documents are hosted privately a URL can be provided along with the `openapi_doc_auth_header` and `openapi_doc_auth_token` inputs.
+          Each document will be fetched using the provided auth header and token, so they need to be valid for all documents.
+
+          For example:
+          overlay_docs: |
+            - https://example.com/overlay1.json
+            - https://example.com/overlay2.json
+        required: false
+        type: string
       languages:
         description: |-
           A yaml string containing a list of languages to generate SDKs for example:
@@ -187,6 +202,7 @@ jobs:
           openapi_doc_auth_header: ${{ inputs.openapi_doc_auth_header }}
           openapi_doc_auth_token: ${{ secrets.openapi_doc_auth_token }}
           openapi_docs: ${{ inputs.openapi_docs }}
+          overlay_docs: ${{ inputs.overlay_docs }}
           github_access_token: ${{ secrets.github_access_token }}
           action: validate
           speakeasy_api_key: ${{ secrets.speakeasy_api_key }}
@@ -245,6 +261,7 @@ jobs:
           openapi_doc_auth_header: ${{ inputs.openapi_doc_auth_header }}
           openapi_doc_auth_token: ${{ secrets.openapi_doc_auth_token }}
           openapi_docs: ${{ inputs.openapi_docs }}
+          overlay_docs: ${{ inputs.overlay_docs }}
           github_access_token: ${{ secrets.github_access_token }}
           languages: ${{ inputs.languages }}
           create_release: ${{ inputs.create_release }}

Makefile

@@ -17,3 +17,6 @@ test-release-mode-multi-sdk:
 
 test-validate-action:
 	./testing/test.sh ./testing/validate-action.env
+
+test-overlay:
+	./testing/test.sh ./testing/overlay-test.env

action.yml

@@ -32,6 +32,19 @@ inputs:
         - https://example.com/openapi1.json
         - https://example.com/openapi2.json
     required: false
+  overlay_docs:
+    description: |-
+      A yaml string containing a list of overlay documents to use, if multiple documents are provided they will be applied to the OpenAPI document in the order provided.
+
+      If the document lives within the repo a relative path can be provided, if the document is hosted publicly a URL can be provided.
+
+      If the documents are hosted privately a URL can be provided along with the `openapi_doc_auth_header` and `openapi_doc_auth_token` inputs.
+      Each document will be fetched using the provided auth header and token, so they need to be valid for all documents.
+
+      For example:
+      overlay_docs: |
+        - https://example.com/overlay1.json
+        - https://example.com/overlay2.json
   openapi_doc_output:
     description: "The path to output the modified OpenAPI spec"
     required: false

internal/cli/cli.go

@@ -2,6 +2,7 @@ package cli
 
 import (
 	"fmt"
+	"os"
 	"regexp"
 	"strconv"
 	"strings"
@@ -18,6 +19,7 @@ var (
 	OutputTestsVersion             = version.Must(version.NewVersion("1.33.2"))
 	LLMSuggestionVersion           = version.Must(version.NewVersion("1.47.1"))
 	GranularChangeLogVersion       = version.Must(version.NewVersion("1.70.2"))
+	OverlayVersion                 = version.Must(version.NewVersion("1.112.1"))
 )
 
 func IsAtLeastVersion(version *version.Version) bool {
@@ -281,3 +283,29 @@ func MergeDocuments(files []string, output string) error {
 	fmt.Println(out)
 	return nil
 }
+
+func ApplyOverlay(overlayPath, inPath, outPath string) error {
+	if !IsAtLeastVersion(OverlayVersion) {
+		return fmt.Errorf("speakeasy version %s does not support applying overlays", OverlayVersion)
+	}
+
+	args := []string{
+		"overlay",
+		"apply",
+		"-o",
+		overlayPath,
+		"-s",
+		inPath,
+	}
+
+	out, err := runSpeakeasyCommand(args...)
+	if err != nil {
+		return fmt.Errorf("error applying overlay: %w - %s", err, out)
+	}
+
+	if err := os.WriteFile(outPath, []byte(out), os.ModePerm); err != nil {
+		return fmt.Errorf("error writing overlay output: %w", err)
+	}
+
+	return nil
+}

internal/document/document.go

@@ -24,29 +24,51 @@ type file struct {
 }
 
 func GetOpenAPIFileInfo() (string, string, string, error) {
-	files, err := getFiles()
+	// TODO OPENAPI_DOC_LOCATION is deprecated and should be removed in the future
+	openapiFiles, err := getFiles(environment.GetOpenAPIDocs(), environment.GetOpenAPIDocLocation())
 	if err != nil {
 		return "", "", "", err
 	}
 
-	if len(files) > 1 && !cli.IsAtLeastVersion(cli.MergeVersion) {
+	if len(openapiFiles) > 1 && !cli.IsAtLeastVersion(cli.MergeVersion) {
 		return "", "", "", fmt.Errorf("multiple openapi files are only supported in speakeasy version %s or higher", cli.MergeVersion.String())
 	}
 
-	filePaths, err := resolveFiles(files)
+	resolvedOpenAPIFiles, err := resolveFiles(openapiFiles, "openapi")
 	if err != nil {
 		return "", "", "", err
 	}
 
 	basePath := ""
 	filePath := ""
 
-	if len(filePaths) == 1 {
-		filePath = filePaths[0]
+	if len(resolvedOpenAPIFiles) == 1 {
+		filePath = resolvedOpenAPIFiles[0]
 		basePath = filepath.Dir(filePath)
 	} else {
-		basePath = filepath.Dir(filePaths[0])
-		filePath, err = mergeFiles(filePaths)
+		basePath = filepath.Dir(resolvedOpenAPIFiles[0])
+		filePath, err = mergeFiles(resolvedOpenAPIFiles)
+		if err != nil {
+			return "", "", "", err
+		}
+	}
+
+	overlayFiles, err := getFiles(environment.GetOverlayDocs(), "")
+	if err != nil {
+		return "", "", "", err
+	}
+
+	if len(overlayFiles) > 1 && !cli.IsAtLeastVersion(cli.OverlayVersion) {
+		return "", "", "", fmt.Errorf("overlay files are only supported in speakeasy version %s or higher", cli.OverlayVersion.String())
+	}
+
+	resolvedOverlayFiles, err := resolveFiles(overlayFiles, "overlay")
+	if err != nil {
+		return "", "", "", err
+	}
+
+	if len(resolvedOverlayFiles) > 0 {
+		filePath, err = applyOverlay(filePath, resolvedOverlayFiles)
 		if err != nil {
 			return "", "", "", err
 		}
@@ -87,7 +109,7 @@ func GetOpenAPIFileInfo() (string, string, string, error) {
 }
 
 func mergeFiles(files []string) (string, error) {
-	outPath := filepath.Join(environment.GetWorkspace(), "repo", "openapi", "openapi_merged")
+	outPath := filepath.Join(environment.GetWorkspace(), "repo", ".openapi", "openapi_merged")
 
 	if err := os.MkdirAll(filepath.Dir(outPath), os.ModePerm); err != nil {
 		return "", fmt.Errorf("failed to create openapi directory: %w", err)
@@ -100,7 +122,25 @@ func mergeFiles(files []string) (string, error) {
 	return outPath, nil
 }
 
-func resolveFiles(files []file) ([]string, error) {
+func applyOverlay(filePath string, overlayFiles []string) (string, error) {
+	outPath := filepath.Join(environment.GetWorkspace(), "repo", ".openapi", "openapi_overlay")
+
+	if err := os.MkdirAll(filepath.Dir(outPath), os.ModePerm); err != nil {
+		return "", fmt.Errorf("failed to create openapi directory: %w", err)
+	}
+
+	for _, overlayFile := range overlayFiles {
+		if err := cli.ApplyOverlay(overlayFile, filePath, outPath); err != nil {
+			return "", fmt.Errorf("failed to apply overlay: %w", err)
+		}
+
+		filePath = outPath
+	}
+
+	return outPath, nil
+}
+
+func resolveFiles(files []file, typ string) ([]string, error) {
 	workspace := environment.GetWorkspace()
 
 	outFiles := []string{}
@@ -109,18 +149,18 @@ func resolveFiles(files []file) ([]string, error) {
 		localPath := filepath.Join(workspace, "repo", file.Location)
 
 		if _, err := os.Stat(localPath); err == nil {
-			fmt.Println("Found local OpenAPI file: ", localPath)
+			fmt.Printf("Found local %s file: %s\n", typ, localPath)
 
 			outFiles = append(outFiles, localPath)
 		} else {
 			u, err := url.Parse(file.Location)
 			if err != nil {
-				return nil, fmt.Errorf("failed to parse openapi url: %w", err)
+				return nil, fmt.Errorf("failed to parse %s url: %w", typ, err)
 			}
 
-			fmt.Println("Downloading openapi file from: ", u.String())
+			fmt.Printf("Downloading %s file from: %s\n", typ, u.String())
 
-			filePath := filepath.Join(environment.GetWorkspace(), "openapi", fmt.Sprintf("openapi_%d", i))
+			filePath := filepath.Join(environment.GetWorkspace(), typ, fmt.Sprintf("%s_%d", typ, i))
 
 			if environment.GetAction() == environment.ActionValidate {
 				if extension := path.Ext(u.Path); extension != "" {
@@ -129,11 +169,11 @@ func resolveFiles(files []file) ([]string, error) {
 			}
 
 			if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil {
-				return nil, fmt.Errorf("failed to create openapi directory: %w", err)
+				return nil, fmt.Errorf("failed to create %s directory: %w", typ, err)
 			}
 
 			if err := download.DownloadFile(u.String(), filePath, file.Header, file.Token); err != nil {
-				return nil, fmt.Errorf("failed to download openapi file: %w", err)
+				return nil, fmt.Errorf("failed to download %s file: %w", typ, err)
 			}
 
 			outFiles = append(outFiles, filePath)
@@ -143,17 +183,14 @@ func resolveFiles(files []file) ([]string, error) {
 	return outFiles, nil
 }
 
-func getFiles() ([]file, error) {
-	docsYaml := environment.GetOpenAPIDocs()
-
+func getFiles(filesYaml string, defaultFile string) ([]file, error) {
 	var fileLocations []string
-	if err := yaml.Unmarshal([]byte(docsYaml), &fileLocations); err != nil {
+	if err := yaml.Unmarshal([]byte(filesYaml), &fileLocations); err != nil {
 		return nil, fmt.Errorf("failed to parse openapi_docs input: %w", err)
 	}
 
-	// TODO OPENAPI_DOC_LOCATION is deprecated and should be removed in the future
-	if len(fileLocations) == 0 {
-		fileLocations = append(fileLocations, environment.GetOpenAPIDocLocation())
+	if len(fileLocations) == 0 && defaultFile != "" {
+		fileLocations = append(fileLocations, defaultFile)
 	}
 
 	files := []file{}

internal/environment/environment.go

@@ -117,6 +117,10 @@ func GetOpenAPIDocs() string {
 	return os.Getenv("INPUT_OPENAPI_DOCS")
 }
 
+func GetOverlayDocs() string {
+	return os.Getenv("INPUT_OVERLAY_DOCS")
+}
+
 func GetOpenAPIDocOutput() string {
 	return os.Getenv("INPUT_OPENAPI_DOC_OUTPUT")
 }

testing/overlay-test.env

@@ -0,0 +1,8 @@
+INPUT_MODE="pr"
+INPUT_ACTION="generate"
+INPUT_LANGUAGES="- go"
+INPUT_OPENAPI_DOCS="[\"base_oas.yaml\"]"
+INPUT_OVERLAY_DOCS="[\"terraform_overlay.yaml\"]"
+GITHUB_REPOSITORY="speakeasy-api/sdk-generation-action-overlay-test"
+INPUT_FORCE=true
+RUN_FINALIZE=true