From c79bf609fe7b43a4a8f004e96e74a29d40455a08 Mon Sep 17 00:00:00 2001 From: Dapeng Zhang Date: Tue, 10 Nov 2020 17:28:18 +0800 Subject: [PATCH] Add generator tool to run in the sdk automation pipeline --- tools/apidiff/delta/delta.go | 2 +- tools/apidiff/exports/package.go | 2 +- tools/generator/autorest/autorest.go | 108 ++++++++ tools/generator/autorest/package.go | 113 ++++++++ tools/generator/autorest/package_test.go | 257 ++++++++++++++++++ tools/generator/changelog/changelog.go | 32 +++ tools/generator/changelog/export.go | 93 +++++++ tools/generator/changelog/git.go | 30 ++ tools/generator/cmd/root.go | 223 +++++++++++++++ tools/generator/main.go | 15 + tools/generator/model/model.go | 92 +++++++ .../testdata/2020-10-29/foo/client.go | 1 + .../testdata/2020-10-29/foo/models.go | 1 + .../testdata/2020-10-29/foo/version.go | 1 + .../testdata/2020-10-30/foo/client.go | 1 + .../testdata/2020-10-30/foo/models.go | 1 + .../testdata/2020-10-30/foo/version.go | 1 + 17 files changed, 971 insertions(+), 2 deletions(-) create mode 100644 tools/generator/autorest/autorest.go create mode 100644 tools/generator/autorest/package.go create mode 100644 tools/generator/autorest/package_test.go create mode 100644 tools/generator/changelog/changelog.go create mode 100644 tools/generator/changelog/export.go create mode 100644 tools/generator/changelog/git.go create mode 100644 tools/generator/cmd/root.go create mode 100644 tools/generator/main.go create mode 100644 tools/generator/model/model.go create mode 100644 tools/generator/testdata/2020-10-29/foo/client.go create mode 100644 tools/generator/testdata/2020-10-29/foo/models.go create mode 100644 tools/generator/testdata/2020-10-29/foo/version.go create mode 100644 tools/generator/testdata/2020-10-30/foo/client.go create mode 100644 tools/generator/testdata/2020-10-30/foo/models.go create mode 100644 tools/generator/testdata/2020-10-30/foo/version.go diff --git a/tools/apidiff/delta/delta.go b/tools/apidiff/delta/delta.go index 9d14a261416b..fc2830796b59 100644 --- a/tools/apidiff/delta/delta.go +++ b/tools/apidiff/delta/delta.go @@ -172,7 +172,7 @@ func GetInterfaceMethods(lhs, rhs exports.Content) map[string]exports.Interface // Signature contains the details of how a type signature changed (e.g. From:"int" To:"string"). type Signature struct { - // From contains the originial signature. + // From contains the original signature. From string `json:"from"` // To contains the new signature. diff --git a/tools/apidiff/exports/package.go b/tools/apidiff/exports/package.go index 5c4c2eb47e3e..4052f54a643b 100644 --- a/tools/apidiff/exports/package.go +++ b/tools/apidiff/exports/package.go @@ -162,7 +162,7 @@ func (pkg Package) buildFunc(ft *ast.FuncType) (f Func) { // appends a to s, comma-delimited style, and returns s appendString := func(s, a string) string { if s != "" { - s += "," + s += ", " } s += a return s diff --git a/tools/generator/autorest/autorest.go b/tools/generator/autorest/autorest.go new file mode 100644 index 000000000000..3578ed614d1e --- /dev/null +++ b/tools/generator/autorest/autorest.go @@ -0,0 +1,108 @@ +package autorest + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "strings" +) + +// Task describes a generation task +type Task struct { + // AbsReadmeMd absolute path of the readme.md file to generate + AbsReadmeMd string +} + +// Execute executes the autorest task, and then invoke the after scripts +// the error returned will be TaskError +func (t *Task) Execute(options Options) error { + if err := t.executeAutorest(options.AutorestArguments); err != nil { + return err + } + + if err := t.executeAfterScript(options.AfterScripts); err != nil { + return err + } + + return nil +} + +func (t *Task) executeAutorest(options []string) error { + arguments := append(options, t.AbsReadmeMd) + c := exec.Command("autorest", arguments...) + log.Printf("Executing autorest with parameters: %+v", arguments) + c.Stdout = os.Stdout + c.Stderr = os.Stderr + c.Start() + if err := c.Wait(); err != nil { + return &TaskError{ + AbsReadmeMd: t.AbsReadmeMd, + Script: "autorest", + Message: err.Error(), + } + } + return nil +} + +func (t *Task) executeAfterScript(afterScripts []string) error { + for _, script := range afterScripts { + log.Printf("Executing after scripts %s...", script) + arguments := strings.Split(script, " ") + c := exec.Command(arguments[0], arguments[1:]...) + output, err := c.CombinedOutput() + if err != nil { + return &TaskError{ + AbsReadmeMd: t.AbsReadmeMd, + Script: script, + Message: string(output), + } + } + } + + return nil +} + +// Options describes the options used in an autorest task +type Options struct { + // AutorestArguments are the optional flags for the autorest tool + AutorestArguments []string + // AfterScripts are the scripts that need to be run after the SDK is generated + AfterScripts []string +} + +// NewOptionsFrom returns a new options from a io.Reader +func NewOptionsFrom(reader io.Reader) (*Options, error) { + b, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + var result Options + if err := json.Unmarshal(b, &result); err != nil { + return nil, err + } + return &result, nil +} + +// String ... +func (o Options) String() string { + b, _ := json.MarshalIndent(o, "", " ") + return string(b) +} + +// TaskError the error returned during an autorest task +type TaskError struct { + // AbsReadmeMd relative path of the readme.md file to generate + AbsReadmeMd string + // Script the script running when the error is thrown + Script string + // Message the error message + Message string +} + +func (r *TaskError) Error() string { + return fmt.Sprintf("autorest task failed for readme file '%s' during '%s': %s", r.AbsReadmeMd, r.Script, r.Message) +} diff --git a/tools/generator/autorest/package.go b/tools/generator/autorest/package.go new file mode 100644 index 000000000000..7b1a342c6cc3 --- /dev/null +++ b/tools/generator/autorest/package.go @@ -0,0 +1,113 @@ +package autorest + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +type ChangedPackagesMap map[string][]string + +func (c *ChangedPackagesMap) addFileToPackage(pkg, file string) { + pkg = strings.ReplaceAll(pkg, "\\", "/") + if _, ok := (*c)[pkg]; !ok { + (*c)[pkg] = []string{} + } + (*c)[pkg] = append((*c)[pkg], file) +} + +func (c *ChangedPackagesMap) String() string { + var r []string + for k, v := range *c { + r = append(r, fmt.Sprintf("%s: %+v", k, v)) + } + return strings.Join(r, "\n") +} + +func (c *ChangedPackagesMap) GetChangedPackages() []string { + var r []string + for k := range *c { + r = append(r, k) + } + return r +} + +// GetChangedPackages get the go SDK packages map from the given changed file list. +// the map returned has the package full path as key, and the changed files in the package as the value. +// This function identify the package by checking if a directory has both a `version.go` file and a `client.go` file. +func GetChangedPackages(changedFiles []string) (ChangedPackagesMap, error) { + changedFiles, err := ExpandChangedDirectories(changedFiles) + if err != nil { + return nil, err + } + r := ChangedPackagesMap{} + for _, file := range changedFiles { + fi, err := os.Stat(file) + if err != nil { + return nil, err + } + path := file + if !fi.IsDir() { + path = filepath.Dir(file) + } + if IsValidPackage(path) { + r.addFileToPackage(path, file) + } + } + return r, nil +} + +// ExpandChangedDirectories expands every directory listed in the array to all its file +func ExpandChangedDirectories(changedFiles []string) ([]string, error) { + var result []string + for _, path := range changedFiles { + fi, err := os.Stat(path) + if err != nil { + return nil, err + } + if fi.IsDir() { + siblings, err := getAllFiles(path) + if err != nil { + return nil, err + } + result = append(result, siblings...) + } else { + result = append(result, path) + } + } + + return result, nil +} + +func getAllFiles(root string) ([]string, error) { + var siblings []string + err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + siblings = append(siblings, strings.ReplaceAll(path, "\\", "/")) + } + return nil + }) + return siblings, err +} + +const ( + clientGo = "client.go" + versionGo = "version.go" +) + +func IsValidPackage(dir string) bool { + client := filepath.Join(dir, clientGo) + version := filepath.Join(dir, versionGo) + // both the above files must exist to return true + if _, err := os.Stat(client); os.IsNotExist(err) { + return false + } + if _, err := os.Stat(version); os.IsNotExist(err) { + return false + } + return true +} diff --git a/tools/generator/autorest/package_test.go b/tools/generator/autorest/package_test.go new file mode 100644 index 000000000000..3938ea315382 --- /dev/null +++ b/tools/generator/autorest/package_test.go @@ -0,0 +1,257 @@ +package autorest + +import ( + "reflect" + "testing" +) + +func TestGetChangedPackages(t *testing.T) { + testData := []struct { + description string + changedFiles []string + expected map[string][]string + }{ + { + description: "one file changed in one package", + changedFiles: []string{ + "../testdata/2020-10-29/foo/models.go", + }, + expected: map[string][]string{ + "../testdata/2020-10-29/foo": { + "../testdata/2020-10-29/foo/models.go", + }, + }, + }, + { + description: "two files changed in one package", + changedFiles: []string{ + "../testdata/2020-10-29/foo/models.go", + "../testdata/2020-10-29/foo/version.go", + }, + expected: map[string][]string{ + "../testdata/2020-10-29/foo": { + "../testdata/2020-10-29/foo/models.go", + "../testdata/2020-10-29/foo/version.go", + }, + }, + }, + { + description: "multiple files changed in two packages", + changedFiles: []string{ + "../testdata/2020-10-29/foo/models.go", + "../testdata/2020-10-29/foo/version.go", + "../testdata/2020-10-30/foo/models.go", + }, + expected: map[string][]string{ + "../testdata/2020-10-29/foo": { + "../testdata/2020-10-29/foo/models.go", + "../testdata/2020-10-29/foo/version.go", + }, + "../testdata/2020-10-30/foo": { + "../testdata/2020-10-30/foo/models.go", + }, + }, + }, + { + description: "one directory untracked and one file changed in one package", + changedFiles: []string{ + "../testdata/2020-10-29/foo", + "../testdata/2020-10-30/foo/models.go", + }, + expected: map[string][]string{ + "../testdata/2020-10-29/foo": { + "../testdata/2020-10-29/foo/client.go", + "../testdata/2020-10-29/foo/models.go", + "../testdata/2020-10-29/foo/version.go", + }, + "../testdata/2020-10-30/foo": { + "../testdata/2020-10-30/foo/models.go", + }, + }, + }, + { + description: "two untracked directories", + changedFiles: []string{ + "../testdata/2020-10-29/foo", + "../testdata/2020-10-30/foo", + }, + expected: map[string][]string{ + "../testdata/2020-10-29/foo": { + "../testdata/2020-10-29/foo/client.go", + "../testdata/2020-10-29/foo/models.go", + "../testdata/2020-10-29/foo/version.go", + }, + "../testdata/2020-10-30/foo": { + "../testdata/2020-10-30/foo/client.go", + "../testdata/2020-10-30/foo/models.go", + "../testdata/2020-10-30/foo/version.go", + }, + }, + }, + { + description: "one untracked directory that contains one packages", + changedFiles: []string{ + "../testdata/2020-10-29", + }, + expected: map[string][]string{ + "../testdata/2020-10-29/foo": { + "../testdata/2020-10-29/foo/client.go", + "../testdata/2020-10-29/foo/models.go", + "../testdata/2020-10-29/foo/version.go", + }, + }, + }, + { + description: "one untracked directory that contains multiple packages", + changedFiles: []string{ + "../testdata/", + }, + expected: map[string][]string{ + "../testdata/2020-10-29/foo": { + "../testdata/2020-10-29/foo/client.go", + "../testdata/2020-10-29/foo/models.go", + "../testdata/2020-10-29/foo/version.go", + }, + "../testdata/2020-10-30/foo": { + "../testdata/2020-10-30/foo/client.go", + "../testdata/2020-10-30/foo/models.go", + "../testdata/2020-10-30/foo/version.go", + }, + }, + }, + } + + for _, c := range testData { + t.Logf("testing %s", c.description) + r, err := GetChangedPackages(c.changedFiles) + if err != nil { + t.Fatalf("unexpected error: %+v", err) + } + if !mapDeepEqual(r, c.expected) { + t.Fatalf("expected %+v, but got %+v", c.expected, r) + } + } +} + +func TestExpandChangedDirectories(t *testing.T) { + testData := []struct { + description string + input []string + expected []string + }{ + { + description: "only files", + input: []string{ + "../testdata/2020-10-29/foo/client.go", + }, + expected: []string{ + "../testdata/2020-10-29/foo/client.go", + }, + }, + { + description: "only directories", + input: []string{ + "../testdata/2020-10-29/foo", + }, + expected: []string{ + "../testdata/2020-10-29/foo/client.go", + "../testdata/2020-10-29/foo/models.go", + "../testdata/2020-10-29/foo/version.go", + }, + }, + { + description: "both directories and files", + input: []string{ + "../testdata/2020-10-29/foo", + "../testdata/2020-10-30/foo/models.go", + }, + expected: []string{ + "../testdata/2020-10-29/foo/client.go", + "../testdata/2020-10-29/foo/models.go", + "../testdata/2020-10-29/foo/version.go", + "../testdata/2020-10-30/foo/models.go", + }, + }, + { + description: "multiple hierarchy of directories but only one sub-directory", + input: []string{ + "../testdata/2020-10-29", + }, + expected: []string{ + "../testdata/2020-10-29/foo/client.go", + "../testdata/2020-10-29/foo/models.go", + "../testdata/2020-10-29/foo/version.go", + }, + }, + { + description: "multiple hierarchy of directories", + input: []string{ + "../testdata", + }, + expected: []string{ + "../testdata/2020-10-29/foo/client.go", + "../testdata/2020-10-29/foo/models.go", + "../testdata/2020-10-29/foo/version.go", + "../testdata/2020-10-30/foo/client.go", + "../testdata/2020-10-30/foo/models.go", + "../testdata/2020-10-30/foo/version.go", + }, + }, + } + + for _, c := range testData { + t.Logf("testing %s", c.description) + r, err := ExpandChangedDirectories(c.input) + if err != nil { + t.Fatalf("unexpected error: %+v", err) + } + if !reflect.DeepEqual(r, c.expected) { + t.Fatalf("expected %v but got %v", c.expected, r) + } + } +} + +// subsetOf return true if m2 is the subset of m1 (every key in m1 exists in m2 and the corresponding values are the same) +func subsetOf(m1, m2 map[string][]string) bool { + for k, v := range m1 { + if v2, ok := m2[k]; !ok || !reflect.DeepEqual(v, v2) { + return false + } + } + return true +} + +func mapDeepEqual(m1, m2 map[string][]string) bool { + return subsetOf(m1, m2) && subsetOf(m2, m1) +} + +func TestIsValidPackage(t *testing.T) { + testData := []struct { + input string + expected bool + }{ + { + input: "../../../services/compute/mgmt/2020-06-01/compute", + expected: true, + }, + { + input: "../../../services/compute/mgmt/2020-06-01", + expected: false, + }, + { + input: "../../../storage", + expected: false, + }, + { + input: "../../../profiles", + expected: false, + }, + } + + for _, c := range testData { + r := IsValidPackage(c.input) + if r != c.expected { + t.Fatalf("expected %v but got %v", c.expected, r) + } + } +} diff --git a/tools/generator/changelog/changelog.go b/tools/generator/changelog/changelog.go new file mode 100644 index 000000000000..9105924fd37a --- /dev/null +++ b/tools/generator/changelog/changelog.go @@ -0,0 +1,32 @@ +package changelog + +import "github.com/Azure/azure-sdk-for-go/tools/apidiff/report" + +type Changelog struct { + PackageName string + NewPackage bool + RemovedPackage bool + Modified *report.Package +} + +func (c Changelog) HasBreakingChanges() bool { + return c.RemovedPackage || (c.Modified != nil && c.Modified.HasBreakingChanges()) +} + +func (c Changelog) String() string { + return c.ToMarkdown() +} + +func (c Changelog) ToMarkdown() string { + if c.NewPackage { + return "This is a new package" + } + if c.RemovedPackage { + return "This package was removed" + } + r := c.Modified.ToMarkdown() + if r == "" { + return "No exported changes" + } + return r +} diff --git a/tools/generator/changelog/export.go b/tools/generator/changelog/export.go new file mode 100644 index 000000000000..f6c6caf89adc --- /dev/null +++ b/tools/generator/changelog/export.go @@ -0,0 +1,93 @@ +package changelog + +import ( + "fmt" + "log" + "os" + + "github.com/Azure/azure-sdk-for-go/tools/apidiff/exports" + "github.com/Azure/azure-sdk-for-go/tools/apidiff/report" +) + +func NewChangelogForPackage(pkgDir string) (c *Changelog, err error) { + // first we need to get the current status of the package + rhs, err := getExportForPackage(pkgDir) + if err != nil { + return nil, err + } + log.Printf("Exports of package '%s': %+v", pkgDir, rhs) + // stash everything and get the previous status of the package + if err := stashEverything(); err != nil { + return nil, err + } + // reset everything when we are done + defer func() { + err = resetEverything() + }() + // get the original state of the package + lhs, err := getExportForPackage(pkgDir) + if err != nil { + return nil, err + } + log.Printf("Exports of original package '%s': %+v", pkgDir, lhs) + return getChangelogForPackage(pkgDir, lhs, rhs) +} + +func stashEverything() error { + if err := gitAddAll(); err != nil { + return err + } + if err := gitStash(); err != nil { + return err + } + return nil +} + +func resetEverything() error { + if err := gitStashPop(); err != nil { + return err + } + if err := gitResetHead(); err != nil { + return err + } + return nil +} + +func getChangelogForPackage(pkgDir string, lhs, rhs *exports.Content) (*Changelog, error) { + if lhs == nil && rhs == nil { + return nil, fmt.Errorf("this package does not exist even after the generation, this should never happen") + } + if lhs == nil { + // the package does not exist before the generation: this is a new package + return &Changelog{ + PackageName: pkgDir, + NewPackage: true, + }, nil + } + if rhs == nil { + // the package no longer exists after the generation: this package was removed + return &Changelog{ + PackageName: pkgDir, + RemovedPackage: true, + }, nil + } + // lhs and rhs are both non-nil + p := report.Generate(*lhs, *rhs, false, false) + return &Changelog{ + PackageName: pkgDir, + Modified: &p, + }, nil +} + +func getExportForPackage(pkgDir string) (*exports.Content, error) { + // The function exports.Get does not handle the circumstance that the package does not exist + // therefore we have to check if it exists and if not exit early to ensure we do not return an error + if _, err := os.Stat(pkgDir); os.IsNotExist(err) { + return nil, nil + } + exp, err := exports.Get(pkgDir) + if err != nil { + return nil, err + } + return &exp, nil +} diff --git a/tools/generator/changelog/git.go b/tools/generator/changelog/git.go new file mode 100644 index 000000000000..f1336a9a02f3 --- /dev/null +++ b/tools/generator/changelog/git.go @@ -0,0 +1,30 @@ +package changelog + +import ( + "log" + "os/exec" +) + +func gitAddAll() error { + log.Printf("Executing `git add *`") + c := exec.Command("git", "add", "*") + return c.Run() +} + +func gitStash() error { + log.Printf("Executing `git stash`") + c := exec.Command("git", "stash") + return c.Run() +} + +func gitStashPop() error { + log.Printf("Executing `git stash pop`") + c := exec.Command("git", "stash", "pop") + return c.Run() +} + +func gitResetHead() error { + log.Printf("Executing `git reset HEAD`") + c := exec.Command("git", "reset", "HEAD") + return c.Run() +} diff --git a/tools/generator/cmd/root.go b/tools/generator/cmd/root.go new file mode 100644 index 000000000000..bed52e57f2ac --- /dev/null +++ b/tools/generator/cmd/root.go @@ -0,0 +1,223 @@ +package cmd + +import ( + "fmt" + "github.com/Azure/azure-sdk-for-go/tools/generator/changelog" + "log" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + + "github.com/Azure/azure-sdk-for-go/tools/generator/autorest" + "github.com/Azure/azure-sdk-for-go/tools/generator/model" + "github.com/spf13/cobra" +) + +const ( + sdkRoot = "azure-sdk-for-go" + defaultOptionPath = "generate_options.json" +) + +// Command ... +func Command() *cobra.Command { + rootCmd := &cobra.Command{ + Use: "generator ", + Args: cobra.ExactArgs(2), + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + log.SetFlags(0) // remove the time stamp prefix + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + optionPath, err := cmd.Flags().GetString("options") + if err != nil { + return err + } + return execute(args[0], args[1], Flags{ + OptionPath: optionPath, + }) + }, + SilenceUsage: true, // this command is used for a pipeline, the usage should never show + } + + flags := rootCmd.Flags() + flags.String("options", defaultOptionPath, "Specify a file with the autorest options") + + return rootCmd +} + +type Flags struct { + OptionPath string +} + +func execute(inputPath, outputPath string, flags Flags) error { + log.Printf("Reading generate input file from '%s'...", inputPath) + input, err := readInputFrom(inputPath) + if err != nil { + return fmt.Errorf("cannot read generate input: %+v", err) + } + log.Printf("Generating using the following GenerateInput...\n%s", input.String()) + output, err := generate(input, flags.OptionPath) + if err != nil { + return fmt.Errorf("cannot generate: %+v", err) + } + log.Printf("Output generated: \n%s", output.String()) + log.Printf("Writing output to file '%s'...", outputPath) + if err := writeOutputTo(outputPath, output); err != nil { + return fmt.Errorf("cannot write generate output: %+v", err) + } + return nil +} + +func readInputFrom(inputPath string) (*model.GenerateInput, error) { + inputFile, err := os.Open(inputPath) + if err != nil { + return nil, err + } + return model.NewGenerateInputFrom(inputFile) +} + +func writeOutputTo(outputPath string, output *model.GenerateOutput) error { + file, err := os.Create(outputPath) + if err != nil { + return err + } + defer file.Close() + if _, err := output.WriteTo(file); err != nil { + return err + } + return nil +} + +// TODO -- support dry run +func generate(input *model.GenerateInput, optionPath string) (*model.GenerateOutput, error) { + if input.DryRun { + return nil, fmt.Errorf("dry run not supported yet") + } + log.Printf("Reading options from file '%s'...", optionPath) + + optionFile, err := os.Open(optionPath) + if err != nil { + return nil, err + } + + options, err := autorest.NewOptionsFrom(optionFile) + if err != nil { + return nil, err + } + log.Printf("Autorest options: \n%s", options.String()) + + // iterate over all the readme + var results []model.PackageResult + for _, readme := range input.RelatedReadmeMdFiles { + log.Printf("Processing readme '%s'...", readme) + task := autorest.Task{ + AbsReadmeMd: filepath.Join(input.SpecFolder, readme), + } + if err := task.Execute(*options); err != nil { + return nil, err + } + // get changed file list + changedFiles, err := getChangedFiles() + if err != nil { + return nil, err + } + log.Printf("Files changed in the SDK: %+v", changedFiles) + // get packages using the changed file list + // returns a map, key is package path, value is files that have changed + packages, err := autorest.GetChangedPackages(changedFiles) + if err != nil { + return nil, err + } + log.Printf("Packages changed: %+v", packages) + // iterate over the changed packages + for p, files := range packages { + log.Printf("Getting package result for package '%s', changed files are: [%s]", p, strings.Join(files, ", ")) + c, err := changelog.NewChangelogForPackage(p) + if err != nil { + return nil, err + } + content := c.ToMarkdown() + breaking := c.HasBreakingChanges() + results = append(results, model.PackageResult{ + PackageName: getPackageIdentifier(p), + Path: []string{p}, + ReadmeMd: []string{readme}, + Changelog: &model.Changelog{ + Content: &content, + HasBreakingChange: &breaking, + }, + }) + } + } + + // sort results + sort.SliceStable(results, func(i, j int) bool { + apiI := getPackageAPIVersionSegment(results[i].PackageName) + apiJ := getPackageAPIVersionSegment(results[j].PackageName) + return apiI > apiJ + }) + + return &model.GenerateOutput{ + Packages: results, + }, nil +} + +func getChangedFiles() ([]string, error) { + var files []string + // get the file changed + changed, err := getDiffFiles() + if err != nil { + return nil, err + } + files = append(files, changed...) + // get the untracked files + untracked, err := getUntrackedFiles() + if err != nil { + return nil, err + } + files = append(files, untracked...) + return files, nil +} + +func getDiffFiles() ([]string, error) { + c := exec.Command("git", "diff", "--name-only") + output, err := c.Output() + if err != nil { + return nil, err + } + var files []string + for _, f := range strings.Split(string(output), "\n") { + f = strings.TrimSpace(f) + if f != "" { + files = append(files, f) + } + } + return files, nil +} + +func getUntrackedFiles() ([]string, error) { + c := exec.Command("git", "ls-files", "--other", "--exclude-standard") + output, err := c.Output() + if err != nil { + return nil, err + } + var files []string + for _, f := range strings.Split(string(output), "\n") { + f = strings.TrimSpace(f) + if f != "" { + files = append(files, f) + } + } + return files, nil +} + +func getPackageIdentifier(pkg string) string { + return strings.TrimPrefix(pkg, "services/") +} + +func getPackageAPIVersionSegment(pkg string) string { + segments := strings.Split(pkg, "/") + return segments[len(segments)-2] +} diff --git a/tools/generator/main.go b/tools/generator/main.go new file mode 100644 index 000000000000..1ef034006d0a --- /dev/null +++ b/tools/generator/main.go @@ -0,0 +1,15 @@ +package main + +import ( + "log" + "os" + + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd" +) + +func main() { + if err := cmd.Command().Execute(); err != nil { + log.Printf("[ERROR] generation meets an error: %+v", err) + os.Exit(1) + } +} diff --git a/tools/generator/model/model.go b/tools/generator/model/model.go new file mode 100644 index 000000000000..f4b283d48d26 --- /dev/null +++ b/tools/generator/model/model.go @@ -0,0 +1,92 @@ +package model + +import ( + "encoding/json" + "io" + "io/ioutil" +) + +// GenerateInput ... +type GenerateInput struct { + DryRun bool `json:"dryRun,omitempty,omitempty"` + SpecFolder string `json:"specFolder,omitempty"` + HeadSha string `json:"headSha,omitempty"` + HeadRef string `json:"headRef,omitempty"` + RepoHttpsUrl string `json:"repoHttpsUrl,omitempty"` + Trigger string `json:"trigger,omitempty"` + ChangedFiles []string `json:"changedFiles,omitempty"` + RelatedReadmeMdFiles []string `json:"relatedReadmeMdFiles,omitempty"` + InstallInstructionInput InstallInstructionScriptInput `json:"installInstructionInput,omitempty"` +} + +// NewGenerateInputFrom ... +func NewGenerateInputFrom(reader io.Reader) (*GenerateInput, error) { + b, err := ioutil.ReadAll(reader) + if err != nil { + return nil, err + } + var result GenerateInput + if err := json.Unmarshal(b, &result); err != nil { + return nil, err + } + return &result, nil +} + +// String ... +func (i GenerateInput) String() string { + b, _ := json.MarshalIndent(i, "", " ") + return string(b) +} + +// InstallInstructionScriptInput ... +type InstallInstructionScriptInput struct { + PackageName string `json:"packageName,omitempty"` + Artifacts []string `json:"artifacts,omitempty"` + IsPublic bool `json:"isPublic,omitempty"` + DownloadUrlPrefix string `json:"downloadUrlPrefix,omitempty"` + DownloadCommandTemplate string `json:"downloadCommandTemplate,omitempty"` + Trigger string `json:"trigger,omitempty"` +} + +// GenerateOutput ... +type GenerateOutput struct { + Packages []PackageResult `json:"packages,omitempty"` +} + +// String ... +func (o GenerateOutput) String() string { + b, _ := json.MarshalIndent(o, "", " ") + return string(b) +} + +// WriteTo ... +func (o GenerateOutput) WriteTo(writer io.Writer) (int64, error) { + b, err := json.Marshal(o) + if err != nil { + return 0, err + } + i, err := writer.Write(b) + return int64(i), err +} + +// PackageResult ... +type PackageResult struct { + PackageName string `json:"packageName,omitempty"` + Path []string `json:"path,omitempty"` + ReadmeMd []string `json:"readmeMd,omitempty"` + Changelog *Changelog `json:"changelog,omitempty"` + Artifacts []string `json:"artifacts,omitempty"` + InstallInstructions *InstallInstructionScriptOutput `json:"installInstructions,omitempty"` +} + +// Changelog ... +type Changelog struct { + Content *string `json:"content,omitempty"` + HasBreakingChange *bool `json:"hasBreakingChange,omitempty"` +} + +// InstallInstructionScriptOutput ... +type InstallInstructionScriptOutput struct { + Full string `json:"full,omitempty"` + Lite string `json:"lite,omitempty"` +} diff --git a/tools/generator/testdata/2020-10-29/foo/client.go b/tools/generator/testdata/2020-10-29/foo/client.go new file mode 100644 index 000000000000..f52652b1ba78 --- /dev/null +++ b/tools/generator/testdata/2020-10-29/foo/client.go @@ -0,0 +1 @@ +package foo diff --git a/tools/generator/testdata/2020-10-29/foo/models.go b/tools/generator/testdata/2020-10-29/foo/models.go new file mode 100644 index 000000000000..f52652b1ba78 --- /dev/null +++ b/tools/generator/testdata/2020-10-29/foo/models.go @@ -0,0 +1 @@ +package foo diff --git a/tools/generator/testdata/2020-10-29/foo/version.go b/tools/generator/testdata/2020-10-29/foo/version.go new file mode 100644 index 000000000000..f52652b1ba78 --- /dev/null +++ b/tools/generator/testdata/2020-10-29/foo/version.go @@ -0,0 +1 @@ +package foo diff --git a/tools/generator/testdata/2020-10-30/foo/client.go b/tools/generator/testdata/2020-10-30/foo/client.go new file mode 100644 index 000000000000..f52652b1ba78 --- /dev/null +++ b/tools/generator/testdata/2020-10-30/foo/client.go @@ -0,0 +1 @@ +package foo diff --git a/tools/generator/testdata/2020-10-30/foo/models.go b/tools/generator/testdata/2020-10-30/foo/models.go new file mode 100644 index 000000000000..f52652b1ba78 --- /dev/null +++ b/tools/generator/testdata/2020-10-30/foo/models.go @@ -0,0 +1 @@ +package foo diff --git a/tools/generator/testdata/2020-10-30/foo/version.go b/tools/generator/testdata/2020-10-30/foo/version.go new file mode 100644 index 000000000000..f52652b1ba78 --- /dev/null +++ b/tools/generator/testdata/2020-10-30/foo/version.go @@ -0,0 +1 @@ +package foo