Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Calculate and check sha256 of plugins #1072

Merged
merged 5 commits into from
May 26, 2023
Merged

Calculate and check sha256 of plugins #1072

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e94d4cb
Calculate and check sha256 of plugins
May 24, 2023
03b1d20
remove checksum for dependencies
May 24, 2023
9baf20c
fix unit test
May 24, 2023
04bcdee
fix lint issue
May 24, 2023
c0d8544
add option to calculate checksum
May 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion hack/gen-plugin-index.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func main() {
})

log.Info("Building index..")
idx, err := idxBuilder.Build(absBinsDir, *urlBasePath, *pluginNameFilter)
idx, err := idxBuilder.Build(absBinsDir, *urlBasePath, *pluginNameFilter, false)
exitOnError("while building plugin index", err)

raw, err := yaml.Marshal(idx)
Expand Down
1 change: 1 addition & 0 deletions internal/plugin/index.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ type (
// IndexURL holds the binary url details.
IndexURL struct {
URL string `yaml:"url"`
Checksum string `yaml:"checksum"`
Platform IndexURLPlatform `yaml:"platform"`
Dependencies Dependencies `yaml:"dependencies,omitempty"`
}
Expand Down
43 changes: 38 additions & 5 deletions internal/plugin/index_builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@ package plugin

import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
Expand Down Expand Up @@ -40,7 +43,7 @@ func NewIndexBuilder(log logrus.FieldLogger) *IndexBuilder {
}

// Build returns plugin index built based on plugins found in a given directory.
func (i *IndexBuilder) Build(dir, urlBasePath, pluginNameFilter string) (Index, error) {
func (i *IndexBuilder) Build(dir, urlBasePath, pluginNameFilter string, skipChecksum bool) (Index, error) {
pluginNameRegex, err := regexp.Compile(pluginNameFilter)
if err != nil {
return Index{}, fmt.Errorf("while compiling filter regex: %w", err)
Expand Down Expand Up @@ -71,6 +74,11 @@ func (i *IndexBuilder) Build(dir, urlBasePath, pluginNameFilter string) (Index,
return Index{}, fmt.Errorf("while getting plugin metadata: %w", err)
}

urls, err := i.mapToIndexURLs(dir, bins, urlBasePath, meta.Dependencies, skipChecksum)
if err != nil {
return Index{}, err
}

pType, pName, _ := strings.Cut(key, "/")
out.Entries = append(out.Entries, IndexEntry{
Name: pName,
Expand All @@ -81,7 +89,7 @@ func (i *IndexBuilder) Build(dir, urlBasePath, pluginNameFilter string) (Index,
Value: meta.JSONSchema.Value,
RefURL: meta.JSONSchema.RefURL,
},
URLs: i.mapToIndexURLs(bins, urlBasePath, meta.Dependencies),
URLs: urls,
})
}

Expand All @@ -94,11 +102,20 @@ func (i *IndexBuilder) Build(dir, urlBasePath, pluginNameFilter string) (Index,
return out, nil
}

func (i *IndexBuilder) mapToIndexURLs(bins []pluginBinariesIndex, urlBasePath string, deps map[string]api.Dependency) []IndexURL {
func (i *IndexBuilder) mapToIndexURLs(parentDir string, bins []pluginBinariesIndex, urlBasePath string, deps map[string]api.Dependency, skipChecksum bool) ([]IndexURL, error) {
var urls []IndexURL
var checksum string
var err error
for _, bin := range bins {
if !skipChecksum {
checksum, err = i.calculateChecksum(filepath.Join(parentDir, bin.BinaryPath))
if err != nil {
return nil, fmt.Errorf("while calculating checksum: %w", err)
}
}
urls = append(urls, IndexURL{
URL: fmt.Sprintf("%s/%s", urlBasePath, bin.BinaryPath),
URL: fmt.Sprintf("%s/%s", urlBasePath, bin.BinaryPath),
Checksum: checksum,
Platform: IndexURLPlatform{
OS: bin.OS,
Arch: bin.Arch,
Expand All @@ -107,7 +124,23 @@ func (i *IndexBuilder) mapToIndexURLs(bins []pluginBinariesIndex, urlBasePath st
})
}

return urls
return urls, nil
}

func (i *IndexBuilder) calculateChecksum(bin string) (string, error) {
if info, err := os.Stat(bin); err != nil || info.IsDir() {
return "", fmt.Errorf("while getting file info: %w", err)
}
file, err := os.Open(filepath.Clean(bin))
if err != nil {
return "", fmt.Errorf("while opening file: %w", err)
}
defer file.Close()
provider := sha256.New()
if _, err := io.Copy(provider, file); err != nil {
return "", fmt.Errorf("while calculating checksum: %w", err)
}
return hex.EncodeToString(provider.Sum(nil)), nil
}

func (*IndexBuilder) dependenciesForBinary(bin pluginBinariesIndex, deps map[string]api.Dependency) Dependencies {
Expand Down
26 changes: 22 additions & 4 deletions internal/plugin/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -439,7 +439,7 @@ func (m *Manager) ensurePluginDownloaded(ctx context.Context, binPath string, in
"dependencyUrl": depURL,
}).Info("Downloading dependency...")

err := DownloadBinary(ctx, depPath, depURL)
err := DownloadBinary(ctx, depPath, URL{URL: depURL})
if err != nil {
return fmt.Errorf("while downloading dependency %q for %q: %w", depName, binPath, err)
}
Expand All @@ -449,7 +449,7 @@ func (m *Manager) ensurePluginDownloaded(ctx context.Context, binPath string, in
}

// DownloadBinary downloads binary into specific destination.
func DownloadBinary(ctx context.Context, destPath, url string) error {
func DownloadBinary(ctx context.Context, destPath string, url URL) error {
dir, filename := filepath.Split(destPath)
err := os.MkdirAll(dir, dirPerms)
if err != nil {
Expand All @@ -461,12 +461,21 @@ func DownloadBinary(ctx context.Context, destPath, url string) error {
return fmt.Errorf("while getting working directory: %w", err)
}

urlWithGoGetterMagicParams := fmt.Sprintf("%s?filename=%s", url, filename)
tmpDestPath := destPath + ".downloading"
if stat, err := os.Stat(tmpDestPath); err == nil && stat.IsDir() {
if err = os.RemoveAll(tmpDestPath); err != nil {
return fmt.Errorf("while deleting temporary directory %q: %w", tmpDestPath, err)
}
}

urlWithGoGetterMagicParams := fmt.Sprintf("%s?filename=%s", url.URL, filename)
if url.Checksum != "" {
urlWithGoGetterMagicParams = fmt.Sprintf("%s&checksum=%s", urlWithGoGetterMagicParams, url.Checksum)
}
getterCli := &getter.Client{
Ctx: ctx,
Src: urlWithGoGetterMagicParams,
Dst: dir,
Dst: tmpDestPath,
Pwd: pwd,
Dir: false,
Mode: getter.ClientModeAny,
Expand All @@ -477,6 +486,15 @@ func DownloadBinary(ctx context.Context, destPath, url string) error {
return fmt.Errorf("while downloading binary from URL %q: %w", url, err)
}

if stat, err := os.Stat(tmpDestPath); err == nil && stat.IsDir() {
tempFileName := filepath.Join(tmpDestPath, filename)
if err = os.Rename(tempFileName, destPath); err != nil {
return fmt.Errorf("while renaming binary %q: %w", tempFileName, err)
}
if err = os.RemoveAll(tmpDestPath); err != nil {
return fmt.Errorf("while deleting temporary directory %q: %w", tmpDestPath, err)
}
}
if stat, err := os.Stat(destPath); err == nil && !stat.IsDir() {
err = os.Chmod(destPath, binPerms)
if err != nil {
Expand Down
16 changes: 12 additions & 4 deletions internal/plugin/store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,16 @@ type (
storeEntry struct {
Description string
Version string
URLs map[string]string
URLs map[string]URL
Dependencies map[string]map[string]string
JSONSchema JSONSchema
}

URL struct {
URL string
Checksum string
}

// storePlugins holds enabled plugins indexed by {repo}/{plugin_name} key.
storePlugins[T any] map[string]enabledPlugins[T]

Expand Down Expand Up @@ -119,12 +124,15 @@ func (s storeRepository) key(repo, name string) string {
return fmt.Sprintf("%s/%s", repo, name)
}

func mapBinaryURLs(in []IndexURL) (map[string]string, map[string]map[string]string) {
out := make(map[string]string)
func mapBinaryURLs(in []IndexURL) (map[string]URL, map[string]map[string]string) {
out := make(map[string]URL)
var deps map[string]map[string]string
for _, item := range in {
key := item.Platform.OS + "/" + item.Platform.Arch
out[key] = item.URL
out[key] = URL{
URL: item.URL,
Checksum: item.Checksum,
}

for depName, dep := range item.Dependencies {
if deps == nil {
Expand Down
70 changes: 35 additions & 35 deletions internal/plugin/store_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,33 +21,33 @@ func TestNewStoreRepository(t *testing.T) {
{
Description: "Kubectl executor plugin.",
Version: "v1.5.0",
URLs: map[string]string{
"darwin/amd64": "https://github.com/kubeshop/botkube/releases/download/v0.27.0/executor_kubectl-darwin-amd64",
"darwin/arm64": "https://github.com/kubeshop/botkube/releases/download/v0.27.0/executor_kubectl-darwin-arm64",
"linux/amd64": "https://github.com/kubeshop/botkube/releases/download/v0.27.0/executor_kubectl-linux-amd64",
"linux/arm64": "https://github.com/kubeshop/botkube/releases/download/v0.27.0/executor_kubectl-linux-arm64",
URLs: map[string]URL{
"darwin/amd64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.27.0/executor_kubectl-darwin-amd64"},
"darwin/arm64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.27.0/executor_kubectl-darwin-arm64"},
"linux/amd64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.27.0/executor_kubectl-linux-amd64"},
"linux/arm64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.27.0/executor_kubectl-linux-arm64"},
},
},
{
Description: "Kubectl executor plugin.",
Version: "v1.0.0",
URLs: map[string]string{
"darwin/amd64": "https://github.com/kubeshop/botkube/releases/download/v0.17.0/executor_kubectl-darwin-amd64",
"darwin/arm64": "https://github.com/kubeshop/botkube/releases/download/v0.17.0/executor_kubectl-darwin-arm64",
"linux/amd64": "https://github.com/kubeshop/botkube/releases/download/v0.17.0/executor_kubectl-linux-amd64",
"linux/arm64": "https://github.com/kubeshop/botkube/releases/download/v0.17.0/executor_kubectl-linux-arm64",
URLs: map[string]URL{
"darwin/amd64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.17.0/executor_kubectl-darwin-amd64"},
"darwin/arm64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.17.0/executor_kubectl-darwin-arm64"},
"linux/amd64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.17.0/executor_kubectl-linux-amd64"},
"linux/arm64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.17.0/executor_kubectl-linux-arm64"},
},
},
},
"botkube/helm": {
{
Description: "Helm is the Botkube executor plugin that allows you to run the Helm CLI commands directly from any communication platform.",
Version: "v0.1.0",
URLs: map[string]string{
"darwin/amd64": "https://github.com/kubeshop/botkube/releases/download/v0.1.0/executor_helm_darwin_amd64",
"darwin/arm64": "https://github.com/kubeshop/botkube/releases/download/v0.1.0/executor_helm_darwin_arm64",
"linux/amd64": "https://github.com/kubeshop/botkube/releases/download/v0.1.0/executor_helm_linux_amd64",
"linux/arm64": "https://github.com/kubeshop/botkube/releases/download/v0.1.0/executor_helm_linux_arm64",
URLs: map[string]URL{
"darwin/amd64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.1.0/executor_helm_darwin_amd64"},
"darwin/arm64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.1.0/executor_helm_darwin_arm64"},
"linux/amd64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.1.0/executor_helm_linux_amd64"},
"linux/arm64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.1.0/executor_helm_linux_arm64"},
},
Dependencies: map[string]map[string]string{
"helm": {
Expand All @@ -63,11 +63,11 @@ func TestNewStoreRepository(t *testing.T) {
{
Description: "Executor suitable for e2e testing. It returns the command that was send as an input.",
Version: "v1.0.2",
URLs: map[string]string{
"darwin/amd64": "https://github.com/mszostok/botkube-plugins/releases/download/v1.0.2/executor_echo-darwin-amd64",
"darwin/arm64": "https://github.com/mszostok/botkube-plugins/releases/download/v1.0.2/executor_echo-darwin-arm64",
"linux/amd64": "https://github.com/mszostok/botkube-plugins/releases/download/v1.0.2/executor_echo-linux-amd64",
"linux/arm64": "https://github.com/mszostok/botkube-plugins/releases/download/v1.0.2/executor_echo-linux-arm64",
URLs: map[string]URL{
"darwin/amd64": {URL: "https://github.com/mszostok/botkube-plugins/releases/download/v1.0.2/executor_echo-darwin-amd64"},
"darwin/arm64": {URL: "https://github.com/mszostok/botkube-plugins/releases/download/v1.0.2/executor_echo-darwin-arm64"},
"linux/amd64": {URL: "https://github.com/mszostok/botkube-plugins/releases/download/v1.0.2/executor_echo-linux-amd64"},
"linux/arm64": {URL: "https://github.com/mszostok/botkube-plugins/releases/download/v1.0.2/executor_echo-linux-arm64"},
},
},
},
Expand All @@ -77,33 +77,33 @@ func TestNewStoreRepository(t *testing.T) {
{
Description: "Kubernetes source",
Version: "v1.0.0",
URLs: map[string]string{
"darwin/amd64": "https://github.com/kubeshop/botkube/releases/download/v0.17.0/darwin_amd64_source_kubernetes",
"darwin/arm64": "https://github.com/kubeshop/botkube/releases/download/v0.17.0/darwin_arm64_source_kubernetes",
"linux/amd64": "https://github.com/kubeshop/botkube/releases/download/v0.17.0/linux-_md64_source_kubernetes",
"linux/arm64": "https://github.com/kubeshop/botkube/releases/download/v0.17.0/linux-_rm64_source_kubernetes",
URLs: map[string]URL{
"darwin/amd64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.17.0/darwin_amd64_source_kubernetes"},
"darwin/arm64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.17.0/darwin_arm64_source_kubernetes"},
"linux/amd64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.17.0/linux-_md64_source_kubernetes"},
"linux/arm64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.17.0/linux-_rm64_source_kubernetes"},
},
},
{
Description: "Kubernetes source",
Version: "0.1.0", // should support also version without `v`
URLs: map[string]string{
"darwin/amd64": "https://github.com/kubeshop/botkube/releases/download/v0.1.0/darwin_amd64_source_kubernetes",
"darwin/arm64": "https://github.com/kubeshop/botkube/releases/download/v0.1.0/darwin_arm64_source_kubernetes",
"linux/amd64": "https://github.com/kubeshop/botkube/releases/download/v0.1.0/linux-_md64_source_kubernetes",
"linux/arm64": "https://github.com/kubeshop/botkube/releases/download/v0.1.0/linux-_rm64_source_kubernetes",
URLs: map[string]URL{
"darwin/amd64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.1.0/darwin_amd64_source_kubernetes"},
"darwin/arm64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.1.0/darwin_arm64_source_kubernetes"},
"linux/amd64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.1.0/linux-_md64_source_kubernetes"},
"linux/arm64": {URL: "https://github.com/kubeshop/botkube/releases/download/v0.1.0/linux-_rm64_source_kubernetes"},
},
},
},
"mszostok/cm-watcher": {
{
Description: "Source suitable for e2e testing.",
Version: "v1.0.0",
URLs: map[string]string{
"darwin/amd64": "https://github.com/mszostok/botkube-plugins/releases/download/v1.0.0/darwin_amd64_cmd-watcher",
"darwin/arm64": "https://github.com/mszostok/botkube-plugins/releases/download/v1.0.0/darwin_arm64_cmd-watcher",
"linux/amd64": "https://github.com/mszostok/botkube-plugins/releases/download/v1.0.0/linux-_md64_cmd-watcher",
"linux/arm64": "https://github.com/mszostok/botkube-plugins/releases/download/v1.0.0/linux-_rm64_cmd-watcher",
URLs: map[string]URL{
"darwin/amd64": {URL: "https://github.com/mszostok/botkube-plugins/releases/download/v1.0.0/darwin_amd64_cmd-watcher"},
"darwin/arm64": {URL: "https://github.com/mszostok/botkube-plugins/releases/download/v1.0.0/darwin_arm64_cmd-watcher"},
"linux/amd64": {URL: "https://github.com/mszostok/botkube-plugins/releases/download/v1.0.0/linux-_md64_cmd-watcher"},
"linux/arm64": {URL: "https://github.com/mszostok/botkube-plugins/releases/download/v1.0.0/linux-_rm64_cmd-watcher"},
},
},
},
Expand Down
3 changes: 1 addition & 2 deletions test/fake/plugin_server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,11 @@ func NewPluginServer(cfg PluginConfig) (string, func() error) {
builder := plugin.NewIndexBuilder(loggerx.NewNoop())

http.HandleFunc(indexFileEndpoint, func(w http.ResponseWriter, _ *http.Request) {
idx, err := builder.Build(cfg.BinariesDirectory, basePath+"/static", ".*")
idx, err := builder.Build(cfg.BinariesDirectory, basePath+"/static", ".*", true)
if err != nil {
log.Printf("Cannot build index file: %s", err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
}

out, err := yaml.Marshal(idx)
if err != nil {
log.Printf("Cannot marshall index file: %s", err.Error())
Expand Down