Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master' into move_prom_to_ga
Browse files Browse the repository at this point in the history
  • Loading branch information
ChrsMark committed Oct 5, 2020
2 parents 01f11d5 + c912167 commit b20ab21
Show file tree
Hide file tree
Showing 42 changed files with 2,597 additions and 224 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -349,6 +349,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add missing info about the rest of the azure metricsets in the documentation. {pull}19601[19601]
- Fix k8s scheduler compatibility issue. {pull}19699[19699]
- Fix SQL module mapping NULL values as string {pull}18955[18955] {issue}18898[18898
- Add support for azure light metricset app_stats. {pull}20639[20639]
- Fix ec2 disk and network metrics to use Sum statistic method. {pull}20680[20680]
- Fill cloud.account.name with accountID if account alias doesn't exist. {pull}20736[20736]
- The Kibana collector applies backoff when errored at getting usage stats {pull}20772[20772]
Expand Down Expand Up @@ -731,6 +732,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Release lambda metricset in aws module as GA. {issue}21251[21251] {pull}21255[21255]
- Add dashboard for pubsub metricset in googlecloud module. {pull}21326[21326] {issue}17137[17137]
- Move Prometheus query & remote_write to GA. {pull}21507[21507]
- Map cloud data filed `cloud.account.id` to azure subscription. {pull}21483[21483] {issue}21381[21381]

*Packetbeat*

Expand Down
146 changes: 57 additions & 89 deletions filebeat/input/filestream/fswatch_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ import (
"github.com/elastic/beats/v7/libbeat/logp"
)

var (
excludedFilePath = filepath.Join("testdata", "excluded_file")
includedFilePath = filepath.Join("testdata", "included_file")
directoryPath = filepath.Join("testdata", "unharvestable_dir")
)

func TestFileScanner(t *testing.T) {
testCases := map[string]struct {
paths []string
Expand All @@ -39,56 +45,30 @@ func TestFileScanner(t *testing.T) {
expectedFiles []string
}{
"select all files": {
paths: []string{
filepath.Join("testdata", "excluded_file"),
filepath.Join("testdata", "included_file"),
},
paths: []string{excludedFilePath, includedFilePath},
expectedFiles: []string{
mustAbsPath(filepath.Join("testdata", "excluded_file")),
mustAbsPath(filepath.Join("testdata", "included_file")),
mustAbsPath(excludedFilePath),
mustAbsPath(includedFilePath),
},
},
"skip excluded files": {
paths: []string{
filepath.Join("testdata", "excluded_file"),
filepath.Join("testdata", "included_file"),
},
paths: []string{excludedFilePath, includedFilePath},
excludedFiles: []match.Matcher{
match.MustCompile(filepath.Join("testdata", "excluded_file")),
},
expectedFiles: []string{
mustAbsPath(filepath.Join("testdata", "included_file")),
},
},
// covers test_input.py/test_skip_symlinks
"skip symlinks": {
paths: []string{
filepath.Join("testdata", "symlink_to_included_file"),
filepath.Join("testdata", "included_file"),
match.MustCompile("excluded_file"),
},
symlinks: false,
expectedFiles: []string{
mustAbsPath(filepath.Join("testdata", "included_file")),
},
},
"return a file once if symlinks are enabled": {
paths: []string{
filepath.Join("testdata", "symlink_to_included_file"),
filepath.Join("testdata", "included_file"),
},
symlinks: true,
expectedFiles: []string{
mustAbsPath(filepath.Join("testdata", "included_file")),
mustAbsPath(includedFilePath),
},
},
"skip directories": {
paths: []string{
filepath.Join("testdata", "unharvestable_dir"),
},
paths: []string{directoryPath},
expectedFiles: []string{},
},
}

setupFilesForScannerTest(t)
defer removeFilesOfScannerTest(t)

for name, test := range testCases {
test := test

Expand All @@ -107,11 +87,50 @@ func TestFileScanner(t *testing.T) {
for p, _ := range files {
paths = append(paths, p)
}
assert.Equal(t, test.expectedFiles, paths)
assert.True(t, checkIfSameContents(test.expectedFiles, paths))
})
}
}

func setupFilesForScannerTest(t *testing.T) {
err := os.MkdirAll(directoryPath, 0750)
if err != nil {
t.Fatal(t)
}

for _, path := range []string{excludedFilePath, includedFilePath} {
f, err := os.Create(path)
if err != nil {
t.Fatalf("file %s, error %v", path, err)
}
f.Close()
}
}

func removeFilesOfScannerTest(t *testing.T) {
err := os.RemoveAll("testdata")
if err != nil {
t.Fatal(err)
}
}

// only handles sets
func checkIfSameContents(one, other []string) bool {
if len(one) != len(other) {
return false
}

mustFind := len(one)
for _, oneElem := range one {
for _, otherElem := range other {
if oneElem == otherElem {
mustFind--
}
}
}
return mustFind == 0
}

func TestFileWatchNewDeleteModified(t *testing.T) {
oldTs := time.Now()
newTs := oldTs.Add(5 * time.Second)
Expand Down Expand Up @@ -201,9 +220,7 @@ func TestFileWatchNewDeleteModified(t *testing.T) {
events: make(chan loginp.FSEvent),
}

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go w.watch(ctx)
go w.watch(context.Background())

for _, expectedEvent := range test.expectedEvents {
evt := w.Event()
Expand All @@ -213,55 +230,6 @@ func TestFileWatchNewDeleteModified(t *testing.T) {
}
}

func TestFileWatcherRenamedFile(t *testing.T) {
testPath := mustAbsPath(filepath.Join("testdata", "first_name"))
renamedPath := mustAbsPath(filepath.Join("testdata", "renamed"))

f, err := os.Create(testPath)
if err != nil {
t.Fatal(err)
}
f.Close()
fi, err := os.Stat(testPath)
if err != nil {
t.Fatal(err)
}

cfg := fileScannerConfig{
ExcludedFiles: nil,
Symlinks: false,
RecursiveGlob: false,
}
scanner, err := newFileScanner([]string{testPath, renamedPath}, cfg)
if err != nil {
t.Fatal(err)
}
w := fileWatcher{
log: logp.L(),
scanner: scanner,
events: make(chan loginp.FSEvent),
}

ctx, cancel := context.WithCancel(context.Background())
defer cancel()

go w.watch(ctx)
assert.Equal(t, loginp.FSEvent{Op: loginp.OpCreate, OldPath: "", NewPath: testPath, Info: fi}, w.Event())

err = os.Rename(testPath, renamedPath)
if err != nil {
t.Fatal(err)
}
defer os.Remove(renamedPath)
fi, err = os.Stat(renamedPath)
if err != nil {
t.Fatal(err)
}

go w.watch(ctx)
assert.Equal(t, loginp.FSEvent{Op: loginp.OpRename, OldPath: testPath, NewPath: renamedPath, Info: fi}, w.Event())
}

type mockScanner struct {
files map[string]os.FileInfo
}
Expand Down
144 changes: 144 additions & 0 deletions filebeat/input/filestream/fswatch_test_non_windows.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

// +build !windows

package filestream

import (
"context"
"os"
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"

loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile"
"github.com/elastic/beats/v7/libbeat/common/match"
"github.com/elastic/beats/v7/libbeat/logp"
)

func TestFileScannerSymlinks(t *testing.T) {
testCases := map[string]struct {
paths []string
excludedFiles []match.Matcher
symlinks bool
expectedFiles []string
}{
// covers test_input.py/test_skip_symlinks
"skip symlinks": {
paths: []string{
filepath.Join("testdata", "symlink_to_included_file"),
filepath.Join("testdata", "included_file"),
},
symlinks: false,
expectedFiles: []string{
mustAbsPath(filepath.Join("testdata", "included_file")),
},
},
"return a file once if symlinks are enabled": {
paths: []string{
filepath.Join("testdata", "symlink_to_included_file"),
filepath.Join("testdata", "included_file"),
},
symlinks: true,
expectedFiles: []string{
mustAbsPath(filepath.Join("testdata", "included_file")),
},
},
}

err := os.Symlink(
mustAbsPath(filepath.Join("testdata", "included_file")),
mustAbsPath(filepath.Join("testdata", "symlink_to_included_file")),
)
if err != nil {
t.Fatal(err)
}

for name, test := range testCases {
test := test

t.Run(name, func(t *testing.T) {
cfg := fileScannerConfig{
ExcludedFiles: test.excludedFiles,
Symlinks: true,
RecursiveGlob: false,
}
fs, err := newFileScanner(test.paths, cfg)
if err != nil {
t.Fatal(err)
}
files := fs.GetFiles()
paths := make([]string, 0)
for p, _ := range files {
paths = append(paths, p)
}
assert.Equal(t, test.expectedFiles, paths)
})
}
}

func TestFileWatcherRenamedFile(t *testing.T) {
testPath := mustAbsPath("first_name")
renamedPath := mustAbsPath("renamed")

f, err := os.Create(testPath)
if err != nil {
t.Fatal(err)
}
f.Close()
fi, err := os.Stat(testPath)
if err != nil {
t.Fatal(err)
}

cfg := fileScannerConfig{
ExcludedFiles: nil,
Symlinks: false,
RecursiveGlob: false,
}
scanner, err := newFileScanner([]string{testPath, renamedPath}, cfg)
if err != nil {
t.Fatal(err)
}
w := fileWatcher{
log: logp.L(),
scanner: scanner,
events: make(chan loginp.FSEvent),
}

go w.watch(context.Background())
assert.Equal(t, loginp.FSEvent{Op: loginp.OpCreate, OldPath: "", NewPath: testPath, Info: fi}, w.Event())

err = os.Rename(testPath, renamedPath)
if err != nil {
t.Fatal(err)
}
defer os.Remove(renamedPath)
fi, err = os.Stat(renamedPath)
if err != nil {
t.Fatal(err)
}

go w.watch(context.Background())
evt := w.Event()

assert.Equal(t, loginp.OpRename, evt.Op)
assert.Equal(t, testPath, evt.OldPath)
assert.Equal(t, renamedPath, evt.NewPath)
}
Empty file.
Empty file.

This file was deleted.

3 changes: 3 additions & 0 deletions libbeat/processors/add_cloud_metadata/provider_azure_vm.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ var azureVMMetadataFetcher = provider{
azHeaders := map[string]string{"Metadata": "true"}
azSchema := func(m map[string]interface{}) common.MapStr {
out, _ := s.Schema{
"account": s.Object{
"id": c.Str("subscriptionId"),
},
"instance": s.Object{
"id": c.Str("vmId"),
"name": c.Str("name"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ const azInstanceIdentityDocument = `{
"sku": "14.04.4-LTS",
"version": "14.04.201605091",
"vmId": "04ab04c3-63de-4709-a9f9-9ab8c0411d5e",
"vmSize": "Standard_D3_v2"
"vmSize": "Standard_D3_v2",
"subscriptionId": "5tfb04c3-63de-4709-a9f9-9ab8c0411d5e"
}`

func initAzureTestServer() *httptest.Server {
Expand Down Expand Up @@ -87,6 +88,9 @@ func TestRetrieveAzureMetadata(t *testing.T) {
"machine": common.MapStr{
"type": "Standard_D3_v2",
},
"account": common.MapStr{
"id": "5tfb04c3-63de-4709-a9f9-9ab8c0411d5e",
},
"region": "eastus2",
},
}
Expand Down
Loading

0 comments on commit b20ab21

Please sign in to comment.