diff --git a/cli/azd/.vscode/cspell-azd-dictionary.txt b/cli/azd/.vscode/cspell-azd-dictionary.txt index 5cfe87c7b43..123739eb409 100644 --- a/cli/azd/.vscode/cspell-azd-dictionary.txt +++ b/cli/azd/.vscode/cspell-azd-dictionary.txt @@ -85,6 +85,7 @@ pflag preinit pulumi pyapp +pyvenv restoreapp retriable rzip diff --git a/cli/azd/internal/repository/initializer.go b/cli/azd/internal/repository/initializer.go index e38529d1522..ab595a4a494 100644 --- a/cli/azd/internal/repository/initializer.go +++ b/cli/azd/internal/repository/initializer.go @@ -81,7 +81,7 @@ func (i *Initializer) Initialize( options := copy.Options{} if skipStagingFiles != nil { - options.Skip = func(src string) (bool, error) { + options.Skip = func(fileInfo os.FileInfo, src, dest string) (bool, error) { if _, shouldSkip := skipStagingFiles[src]; shouldSkip { return true, nil } diff --git a/cli/azd/pkg/project/framework_service_npm.go b/cli/azd/pkg/project/framework_service_npm.go index 03e36f91bf6..7a6902947de 100644 --- a/cli/azd/pkg/project/framework_service_npm.go +++ b/cli/azd/pkg/project/framework_service_npm.go @@ -14,7 +14,6 @@ import ( "github.com/azure/azure-dev/cli/azd/pkg/exec" "github.com/azure/azure-dev/cli/azd/pkg/tools" "github.com/azure/azure-dev/cli/azd/pkg/tools/npm" - "github.com/otiai10/copy" ) type npmProject struct { @@ -90,11 +89,15 @@ func (np *npmProject) Build( } task.SetProgress(NewServiceProgress("Copying deployment package")) - if err := copy.Copy( + + if err := buildForZip( publishSource, publishRoot, - skipPatterns( - filepath.Join(publishSource, "node_modules"), filepath.Join(publishSource, ".azure"))); err != nil { + buildForZipOptions{ + excludeConditions: []excludeDirEntryCondition{ + excludeNodeModules, + }, + }); err != nil { task.SetError(fmt.Errorf("publishing for %s: %w", serviceConfig.Name, err)) return } @@ -106,3 +109,9 @@ func (np *npmProject) Build( }, ) } + +const cNodeModulesName = "node_modules" + +func excludeNodeModules(path string, file os.FileInfo) bool { + return !file.IsDir() && file.Name() == cNodeModulesName +} diff --git a/cli/azd/pkg/project/framework_service_python.go b/cli/azd/pkg/project/framework_service_python.go index fad8792d343..da9bdb1e664 100644 --- a/cli/azd/pkg/project/framework_service_python.go +++ b/cli/azd/pkg/project/framework_service_python.go @@ -16,7 +16,6 @@ import ( "github.com/azure/azure-dev/cli/azd/pkg/exec" "github.com/azure/azure-dev/cli/azd/pkg/tools" "github.com/azure/azure-dev/cli/azd/pkg/tools/python" - "github.com/otiai10/copy" ) type pythonProject struct { @@ -107,12 +106,15 @@ func (pp *pythonProject) Build( task.SetProgress(NewServiceProgress("Copying deployment package")) - if err := copy.Copy( + if err := buildForZip( publishSource, publishRoot, - skipPatterns( - filepath.Join(publishSource, "__pycache__"), filepath.Join(publishSource, ".venv"), - filepath.Join(publishSource, ".azure"))); err != nil { + buildForZipOptions{ + excludeConditions: []excludeDirEntryCondition{ + excludeVirtualEnv, + excludePyCache, + }, + }); err != nil { task.SetError(fmt.Errorf("publishing for %s: %w", serviceConfig.Name, err)) return } @@ -125,6 +127,29 @@ func (pp *pythonProject) Build( ) } +const cVenvConfigFileName = "pyvenv.cfg" + +func excludeVirtualEnv(path string, file os.FileInfo) bool { + if !file.IsDir() { + return false + } + + // check if `pyvenv.cfg` is within the folder + if _, err := os.Stat(filepath.Join(path, cVenvConfigFileName)); err == nil { + return true + } + return false +} + +func excludePyCache(path string, file os.FileInfo) bool { + if !file.IsDir() { + return false + } + + folderName := strings.ToLower(file.Name()) + return folderName == "__pycache__" +} + func (pp *pythonProject) getVenvName(serviceConfig *ServiceConfig) string { trimmedPath := strings.TrimSpace(serviceConfig.Path()) if len(trimmedPath) > 0 && trimmedPath[len(trimmedPath)-1] == os.PathSeparator { @@ -133,23 +158,3 @@ func (pp *pythonProject) getVenvName(serviceConfig *ServiceConfig) string { _, projectDir := filepath.Split(trimmedPath) return projectDir + "_env" } - -// skipPatterns returns a `copy.Options` which will skip any files -// that match a given pattern. Matching is done with `filepath.Match`. -func skipPatterns(patterns ...string) copy.Options { - return copy.Options{ - Skip: func(src string) (bool, error) { - for _, pattern := range patterns { - skip, err := filepath.Match(pattern, src) - switch { - case err != nil: - return false, fmt.Errorf("error matching pattern %s: %w", pattern, err) - case skip: - return true, nil - } - } - - return false, nil - }, - } -} diff --git a/cli/azd/pkg/project/internal/project_utils.go b/cli/azd/pkg/project/internal/project_utils.go deleted file mode 100644 index a2d1a356521..00000000000 --- a/cli/azd/pkg/project/internal/project_utils.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -package internal - -import ( - "fmt" - "os" - - "github.com/azure/azure-dev/cli/azd/pkg/rzip" -) - -// CreateDeployableZip creates a zip file of a folder, recursively. -// Returns the path to the created zip file or an error if it fails. -func CreateDeployableZip(appName string, path string) (string, error) { - // TODO: should probably avoid picking up files that weren't meant to be published (ie, local .env files, etc..) - zipFile, err := os.CreateTemp("", "azddeploy*.zip") - if err != nil { - return "", fmt.Errorf("failed when creating zip package to deploy %s: %w", appName, err) - } - - if err := rzip.CreateFromDirectory(path, zipFile); err != nil { - // if we fail here just do our best to close things out and cleanup - zipFile.Close() - os.Remove(zipFile.Name()) - return "", err - } - - if err := zipFile.Close(); err != nil { - // may fail but, again, we'll do our best to cleanup here. - os.Remove(zipFile.Name()) - return "", err - } - - return zipFile.Name(), nil -} diff --git a/cli/azd/pkg/project/project_utils.go b/cli/azd/pkg/project/project_utils.go new file mode 100644 index 00000000000..07ebb2edcf6 --- /dev/null +++ b/cli/azd/pkg/project/project_utils.go @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package project + +import ( + "fmt" + "os" + + "github.com/azure/azure-dev/cli/azd/pkg/rzip" + "github.com/otiai10/copy" +) + +// CreateDeployableZip creates a zip file of a folder, recursively. +// Returns the path to the created zip file or an error if it fails. +func createDeployableZip(appName string, path string) (string, error) { + // TODO: should probably avoid picking up files that weren't meant to be published (ie, local .env files, etc..) + zipFile, err := os.CreateTemp("", "azddeploy*.zip") + if err != nil { + return "", fmt.Errorf("failed when creating zip package to deploy %s: %w", appName, err) + } + + if err := rzip.CreateFromDirectory(path, zipFile); err != nil { + // if we fail here just do our best to close things out and cleanup + zipFile.Close() + os.Remove(zipFile.Name()) + return "", err + } + + if err := zipFile.Close(); err != nil { + // may fail but, again, we'll do our best to cleanup here. + os.Remove(zipFile.Name()) + return "", err + } + + return zipFile.Name(), nil +} + +// excludeDirEntryCondition resolves when a file or directory should be considered or not as part of build, when build is a +// copy-paste source strategy. Return true to exclude the directory entry. +type excludeDirEntryCondition func(path string, file os.FileInfo) bool + +// buildForZipOptions provides a set of options for doing build for zip +type buildForZipOptions struct { + excludeConditions []excludeDirEntryCondition +} + +// buildForZip is use by projects which build strategy is to only copy the source code into a folder which is later +// zipped for packaging. For example Python and Node framework languages. buildForZipOptions provides the specific +// details for each language which should not be ever copied. +func buildForZip(src, dst string, options buildForZipOptions) error { + + // these exclude conditions applies to all projects + options.excludeConditions = append(options.excludeConditions, globalExcludeAzdFolder) + + return copy.Copy(src, dst, copy.Options{ + Skip: func(srcInfo os.FileInfo, src, dest string) (bool, error) { + for _, checkExclude := range options.excludeConditions { + if checkExclude(src, srcInfo) { + return true, nil + } + } + return false, nil + }, + }) +} + +func globalExcludeAzdFolder(path string, file os.FileInfo) bool { + return file.IsDir() && file.Name() == ".azure" +} diff --git a/cli/azd/pkg/project/service_target_appservice.go b/cli/azd/pkg/project/service_target_appservice.go index e231ded7b38..fdc84f26532 100644 --- a/cli/azd/pkg/project/service_target_appservice.go +++ b/cli/azd/pkg/project/service_target_appservice.go @@ -13,7 +13,6 @@ import ( "github.com/azure/azure-dev/cli/azd/pkg/azure" "github.com/azure/azure-dev/cli/azd/pkg/environment" "github.com/azure/azure-dev/cli/azd/pkg/infra" - "github.com/azure/azure-dev/cli/azd/pkg/project/internal" "github.com/azure/azure-dev/cli/azd/pkg/tools" "github.com/azure/azure-dev/cli/azd/pkg/tools/azcli" ) @@ -54,7 +53,7 @@ func (st *appServiceTarget) Package( return async.RunTaskWithProgress( func(task *async.TaskContextWithProgress[*ServicePackageResult, ServiceProgress]) { task.SetProgress(NewServiceProgress("Compressing deployment artifacts")) - zipFilePath, err := internal.CreateDeployableZip(serviceConfig.Name, buildOutput.BuildOutputPath) + zipFilePath, err := createDeployableZip(serviceConfig.Name, buildOutput.BuildOutputPath) if err != nil { task.SetError(err) return diff --git a/cli/azd/pkg/project/service_target_functionapp.go b/cli/azd/pkg/project/service_target_functionapp.go index 81edf7d05b5..f3eb6b54218 100644 --- a/cli/azd/pkg/project/service_target_functionapp.go +++ b/cli/azd/pkg/project/service_target_functionapp.go @@ -13,7 +13,6 @@ import ( "github.com/azure/azure-dev/cli/azd/pkg/azure" "github.com/azure/azure-dev/cli/azd/pkg/environment" "github.com/azure/azure-dev/cli/azd/pkg/infra" - "github.com/azure/azure-dev/cli/azd/pkg/project/internal" "github.com/azure/azure-dev/cli/azd/pkg/tools" "github.com/azure/azure-dev/cli/azd/pkg/tools/azcli" ) @@ -55,7 +54,7 @@ func (f *functionAppTarget) Package( return async.RunTaskWithProgress( func(task *async.TaskContextWithProgress[*ServicePackageResult, ServiceProgress]) { task.SetProgress(NewServiceProgress("Compressing deployment artifacts")) - zipFilePath, err := internal.CreateDeployableZip(serviceConfig.Name, buildOutput.BuildOutputPath) + zipFilePath, err := createDeployableZip(serviceConfig.Name, buildOutput.BuildOutputPath) if err != nil { task.SetError(err) return diff --git a/go.mod b/go.mod index 27e1977323d..d239847c633 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/mattn/go-isatty v0.0.14 github.com/microsoft/ApplicationInsights-Go v0.4.4 github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5 - github.com/otiai10/copy v1.7.0 + github.com/otiai10/copy v1.9.0 github.com/sethvargo/go-retry v0.2.3 github.com/spf13/cobra v1.3.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index a048f1b13df..2dcf74bbfe1 100644 --- a/go.sum +++ b/go.sum @@ -138,7 +138,6 @@ github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWH github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -331,9 +330,8 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.4 h1:5Myjjh3JY/NaAi4IsUbHADytDyl1VE1Y9PXDlL+P/VQ= github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= @@ -383,13 +381,13 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/otiai10/copy v1.7.0 h1:hVoPiN+t+7d2nzzwMiDHPSOogsWAStewq3TwU05+clE= -github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U= +github.com/otiai10/copy v1.9.0 h1:7KFNiCgZ91Ru4qW4CWPf/7jqtxLagGRmIxWldPP9VY4= +github.com/otiai10/copy v1.9.0/go.mod h1:hsfX19wcn0UWIHUQ3/4fHuehhk2UyArQ9dVFAn3FczI= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= -github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI= -github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= +github.com/otiai10/mint v1.4.0 h1:umwcf7gbpEwf7WFzqmWwSv0CzbeMsae2u9ZvpP8j2q4= +github.com/otiai10/mint v1.4.0/go.mod h1:gifjb2MYOoULtKLqUAEILUG/9KONW6f7YsJ6vQLTlFI= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -684,6 +682,7 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=