Add generator tool to run in the sdk automation pipeline

generate_options.json

@@ -0,0 +1,15 @@
+{
+  "autorestArguments": [
+    "[email protected]/autorest.go@~2.1.159",
+    "--go",
+    "--verbose",
+    "--go-sdk-folder=.",
+    "--multiapi",
+    "--use-onever",
+    "--preview-chk",
+    "--version=V2"
+  ],
+  "afterScripts": [
+    "gofmt -w ./services/"
+  ]
+}

tools/apidiff/delta/delta.go

@@ -172,7 +172,7 @@ func GetInterfaceMethods(lhs, rhs exports.Content) map[string]exports.Interface
 
 // Signature contains the details of how a type signature changed (e.g. From:"int" To:"string").
 type Signature struct {
-	// From contains the originial signature.
+	// From contains the original signature.
 	From string `json:"from"`
 
 	// To contains the new signature.

tools/apidiff/delta/delta_test.go

@@ -73,11 +73,11 @@ func Test_GetAddedExports(t *testing.T) {
 
 	fAdded := map[string]exports.Func{
 		"DoNothing2":                 {},
-		"Client.ExportData":          {Params: strPtr("context.Context,string,string,ExportRDBParameters"), Returns: strPtr("ExportDataFuture,error")},
-		"Client.ExportDataPreparer":  {Params: strPtr("context.Context,string,string,ExportRDBParameters"), Returns: strPtr("*http.Request,error")},
-		"Client.ExportDataSender":    {Params: strPtr("*http.Request"), Returns: strPtr("ExportDataFuture,error")},
-		"Client.ExportDataResponder": {Params: strPtr("*http.Response"), Returns: strPtr("autorest.Response,error")},
-		"ExportDataFuture.Result":    {Params: strPtr("Client"), Returns: strPtr("autorest.Response,error")},
+		"Client.ExportData":          {Params: strPtr("context.Context, string, string, ExportRDBParameters"), Returns: strPtr("ExportDataFuture, error")},
+		"Client.ExportDataPreparer":  {Params: strPtr("context.Context, string, string, ExportRDBParameters"), Returns: strPtr("*http.Request, error")},
+		"Client.ExportDataSender":    {Params: strPtr("*http.Request"), Returns: strPtr("ExportDataFuture, error")},
+		"Client.ExportDataResponder": {Params: strPtr("*http.Response"), Returns: strPtr("autorest.Response, error")},
+		"ExportDataFuture.Result":    {Params: strPtr("Client"), Returns: strPtr("autorest.Response, error")},
 	}
 
 	for k, v := range fAdded {
@@ -105,7 +105,7 @@ func Test_GetAddedExports(t *testing.T) {
 			"Two": {Returns: strPtr("error")},
 		}},
 		"SomeInterface": {Methods: map[string]exports.Func{
-			"NewMethod": {Params: strPtr("string"), Returns: strPtr("bool,error")},
+			"NewMethod": {Params: strPtr("string"), Returns: strPtr("bool, error")},
 		}},
 	}
 
@@ -199,7 +199,7 @@ func Test_GetAddedInterfaceMethods(t *testing.T) {
 	added := map[string]exports.Interface{
 		"SomeInterface": {
 			Methods: map[string]exports.Func{
-				"NewMethod": {Params: strPtr("string"), Returns: strPtr("bool,error")},
+				"NewMethod": {Params: strPtr("string"), Returns: strPtr("bool, error")},
 			},
 		},
 	}
@@ -275,17 +275,17 @@ func Test_GetFuncSigChanges(t *testing.T) {
 			Params: &delta.Signature{From: "int", To: delta.None},
 		},
 		"Client.List": {
-			Params:  &delta.Signature{From: "context.Context", To: "context.Context,string"},
-			Returns: &delta.Signature{From: "ListResultPage,error", To: "ListResult,error"},
+			Params:  &delta.Signature{From: "context.Context", To: "context.Context, string"},
+			Returns: &delta.Signature{From: "ListResultPage, error", To: "ListResult, error"},
 		},
 		"Client.ListPreparer": {
-			Params: &delta.Signature{From: "context.Context", To: "context.Context,string"},
+			Params: &delta.Signature{From: "context.Context", To: "context.Context, string"},
 		},
 		"Client.Delete": {
-			Params: &delta.Signature{From: "context.Context,string,string", To: "context.Context,string"},
+			Params: &delta.Signature{From: "context.Context, string, string", To: "context.Context, string"},
 		},
 		"Client.DeletePreparer": {
-			Params: &delta.Signature{From: "context.Context,string,string", To: "context.Context,string"},
+			Params: &delta.Signature{From: "context.Context, string, string", To: "context.Context, string"},
 		},
 	}
 
@@ -316,7 +316,7 @@ func Test_GetInterfaceMethodSigChanges(t *testing.T) {
 		"SomeInterface": {
 			MethodSigs: map[string]delta.FuncSig{
 				"One": {Params: &delta.Signature{From: delta.None, To: "string"}},
-				"Two": {Params: &delta.Signature{From: "bool", To: "bool,int"}},
+				"Two": {Params: &delta.Signature{From: "bool", To: "bool, int"}},
 			},
 		},
 	}

tools/apidiff/exports/package.go

@@ -162,7 +162,7 @@ func (pkg Package) buildFunc(ft *ast.FuncType) (f Func) {
 	// appends a to s, comma-delimited style, and returns s
 	appendString := func(s, a string) string {
 		if s != "" {
-			s += ","
+			s += ", "
 		}
 		s += a
 		return s

tools/apidiff/exports/package_test.go

@@ -73,8 +73,8 @@ func Test_Funcs(t *testing.T) {
 		{"DoNothing", exports.Func{}},
 		{"DoNothingWithParam", exports.Func{Params: strPtr("int"), Returns: nil}},
 		{"UserAgent", exports.Func{Params: nil, Returns: strPtr("string")}},
-		{"Client.Delete", exports.Func{Params: strPtr("context.Context,string,string"), Returns: strPtr("DeleteFuture,error")}},
-		{"Client.ListSender", exports.Func{Params: strPtr("*http.Request"), Returns: strPtr("*http.Response,error")}},
+		{"Client.Delete", exports.Func{Params: strPtr("context.Context, string, string"), Returns: strPtr("DeleteFuture, error")}},
+		{"Client.ListSender", exports.Func{Params: strPtr("*http.Request"), Returns: strPtr("*http.Response, error")}},
 	}
 
 	for _, test := range tests {
@@ -121,8 +121,8 @@ func Test_Interfaces(t *testing.T) {
 					Returns: strPtr("error"),
 				},
 				"Five": {
-					Params:  strPtr("int,bool"),
-					Returns: strPtr("int,error"),
+					Params:  strPtr("int, bool"),
+					Returns: strPtr("int, error"),
 				},
 			},
 		}},

tools/generator/autorest/autorest.go

@@ -0,0 +1,108 @@
+package autorest
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+// Task describes a generation task
+type Task struct {
+	// AbsReadmeMd absolute path of the readme.md file to generate
+	AbsReadmeMd string
+}
+
+// Execute executes the autorest task, and then invoke the after scripts
+// the error returned will be TaskError
+func (t *Task) Execute(options Options) error {
+	if err := t.executeAutorest(options.AutorestArguments); err != nil {
+		return err
+	}
+
+	if err := t.executeAfterScript(options.AfterScripts); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (t *Task) executeAutorest(options []string) error {
+	arguments := append(options, t.AbsReadmeMd)
+	c := exec.Command("autorest", arguments...)
+	log.Printf("Executing autorest with parameters: %+v", arguments)
+	c.Stdout = os.Stdout
+	c.Stderr = os.Stderr
+	c.Start()
+	if err := c.Wait(); err != nil {
+		return &TaskError{
+			AbsReadmeMd: t.AbsReadmeMd,
+			Script:      "autorest",
+			Message:     err.Error(),
+		}
+	}
+	return nil
+}
+
+func (t *Task) executeAfterScript(afterScripts []string) error {
+	for _, script := range afterScripts {
+		log.Printf("Executing after scripts %s...", script)
+		arguments := strings.Split(script, " ")
+		c := exec.Command(arguments[0], arguments[1:]...)
+		output, err := c.CombinedOutput()
+		if err != nil {
+			return &TaskError{
+				AbsReadmeMd: t.AbsReadmeMd,
+				Script:      script,
+				Message:     string(output),
+			}
+		}
+	}
+
+	return nil
+}
+
+// Options describes the options used in an autorest task
+type Options struct {
+	// AutorestArguments are the optional flags for the autorest tool
+	AutorestArguments []string
+	// AfterScripts are the scripts that need to be run after the SDK is generated
+	AfterScripts []string
+}
+
+// NewOptionsFrom returns a new options from a io.Reader
+func NewOptionsFrom(reader io.Reader) (*Options, error) {
+	b, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return nil, err
+	}
+	var result Options
+	if err := json.Unmarshal(b, &result); err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+// String ...
+func (o Options) String() string {
+	b, _ := json.MarshalIndent(o, "", "  ")
+	return string(b)
+}
+
+// TaskError the error returned during an autorest task
+type TaskError struct {
+	// AbsReadmeMd relative path of the readme.md file to generate
+	AbsReadmeMd string
+	// Script the script running when the error is thrown
+	Script string
+	// Message the error message
+	Message string
+}
+
+func (r *TaskError) Error() string {
+	return fmt.Sprintf("autorest task failed for readme file '%s' during '%s': %s", r.AbsReadmeMd, r.Script, r.Message)
+}

tools/generator/autorest/package.go

@@ -0,0 +1,109 @@
+package autorest
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// ChangedPackagesMap is a wrapper of a map of packages. The key is package names, and the value is the changed file list.
+type ChangedPackagesMap map[string][]string
+
+func (c *ChangedPackagesMap) addFileToPackage(pkg, file string) {
+	pkg = strings.ReplaceAll(pkg, "\\", "/")
+	if _, ok := (*c)[pkg]; !ok {
+		(*c)[pkg] = []string{}
+	}
+	(*c)[pkg] = append((*c)[pkg], file)
+}
+
+func (c *ChangedPackagesMap) String() string {
+	var r []string
+	for k, v := range *c {
+		r = append(r, fmt.Sprintf("%s: %+v", k, v))
+	}
+	return strings.Join(r, "\n")
+}
+
+// GetChangedPackages get the go SDK packages map from the given changed file list.
+// the map returned has the package full path as key, and the changed files in the package as the value.
+// This function identify the package by checking if a directory has both a `version.go` file and a `client.go` file.
+func GetChangedPackages(changedFiles []string) (ChangedPackagesMap, error) {
+	changedFiles, err := ExpandChangedDirectories(changedFiles)
+	if err != nil {
+		return nil, err
+	}
+	r := ChangedPackagesMap{}
+	for _, file := range changedFiles {
+		fi, err := os.Stat(file)
+		if err != nil {
+			return nil, err
+		}
+		path := file
+		if !fi.IsDir() {
+			path = filepath.Dir(file)
+		}
+		if IsValidPackage(path) {
+			r.addFileToPackage(path, file)
+		}
+	}
+	return r, nil
+}
+
+// ExpandChangedDirectories expands every directory listed in the array to all its file
+func ExpandChangedDirectories(changedFiles []string) ([]string, error) {
+	var result []string
+	for _, path := range changedFiles {
+		fi, err := os.Stat(path)
+		if err != nil {
+			return nil, err
+		}
+		if fi.IsDir() {
+			siblings, err := getAllFiles(path)
+			if err != nil {
+				return nil, err
+			}
+			result = append(result, siblings...)
+		} else {
+			result = append(result, path)
+		}
+	}
+
+	return result, nil
+}
+
+func getAllFiles(root string) ([]string, error) {
+	var siblings []string
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if !info.IsDir() {
+			siblings = append(siblings, strings.ReplaceAll(path, "\\", "/"))
+		}
+		return nil
+	})
+	return siblings, err
+}
+
+const (
+	clientGo  = "client.go"
+	versionGo = "version.go"
+)
+
+// IsValidPackage returns true when the given directory is a valid azure-sdk-for-go package,
+// otherwise returns false (including the directory does not exist)
+// The criteria of a valid azure-sdk-for-go package is that the directory must have a `client.go` and a `version.go` file
+func IsValidPackage(dir string) bool {
+	client := filepath.Join(dir, clientGo)
+	version := filepath.Join(dir, versionGo)
+	// both the above files must exist to return true
+	if _, err := os.Stat(client); os.IsNotExist(err) {
+		return false
+	}
+	if _, err := os.Stat(version); os.IsNotExist(err) {
+		return false
+	}
+	return true
+}