From b78e9c119e7c1a6d7865cf8519c636706a17eba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Chigot?= Date: Thu, 23 Jan 2020 17:02:29 +0100 Subject: [PATCH] metricbeat: add system metric support for AIX --- libbeat/cmd/instance/metrics.go | 2 +- libbeat/cmd/instance/metrics_other.go | 2 +- libbeat/metric/system/cpu/cpu.go | 2 +- libbeat/metric/system/cpu/cpu_test.go | 2 +- libbeat/metric/system/memory/memory.go | 2 +- libbeat/metric/system/memory/memory_test.go | 2 +- libbeat/metric/system/process/process.go | 2 +- libbeat/metric/system/process/process_test.go | 2 +- metricbeat/module/system/core/core.go | 2 +- metricbeat/module/system/core/core_test.go | 2 +- metricbeat/module/system/cpu/cpu.go | 2 +- metricbeat/module/system/cpu/cpu_test.go | 2 +- .../module/system/filesystem/filesystem.go | 2 +- .../system/filesystem/filesystem_test.go | 2 +- metricbeat/module/system/filesystem/helper.go | 2 +- .../module/system/filesystem/helper_test.go | 2 +- metricbeat/module/system/fsstat/fsstat.go | 2 +- .../module/system/fsstat/fsstat_test.go | 2 +- metricbeat/module/system/load/load.go | 2 +- metricbeat/module/system/load/load_test.go | 2 +- metricbeat/module/system/memory/memory.go | 2 +- .../module/system/memory/memory_test.go | 2 +- metricbeat/module/system/network/network.go | 2 +- .../module/system/network/network_test.go | 2 +- metricbeat/module/system/process/process.go | 2 +- .../module/system/process/process_test.go | 2 +- .../system/process_summary/process_summary.go | 2 +- .../process_summary/process_summary_test.go | 2 +- metricbeat/module/system/test_system.py | 26 +-- metricbeat/module/system/uptime/uptime.go | 2 +- .../module/system/uptime/uptime_test.go | 2 +- metricbeat/tests/system/test_base.py | 2 +- metricbeat/tests/system/test_processors.py | 2 +- metricbeat/tests/system/test_reload.py | 6 +- vendor/github.com/coreos/bbolt/Makefile | 16 +- vendor/github.com/coreos/bbolt/README.md | 176 +++++++++-------- vendor/github.com/coreos/bbolt/bolt_386.go | 2 +- vendor/github.com/coreos/bbolt/bolt_amd64.go | 2 +- vendor/github.com/coreos/bbolt/bolt_arm.go | 2 +- vendor/github.com/coreos/bbolt/bolt_arm64.go | 2 +- vendor/github.com/coreos/bbolt/bolt_linux.go | 2 +- .../github.com/coreos/bbolt/bolt_mips64x.go | 2 +- vendor/github.com/coreos/bbolt/bolt_mipsx.go | 2 +- .../github.com/coreos/bbolt/bolt_openbsd.go | 2 +- vendor/github.com/coreos/bbolt/bolt_ppc.go | 2 +- vendor/github.com/coreos/bbolt/bolt_ppc64.go | 2 +- .../github.com/coreos/bbolt/bolt_ppc64le.go | 2 +- .../github.com/coreos/bbolt/bolt_riscv64.go | 12 ++ vendor/github.com/coreos/bbolt/bolt_s390x.go | 2 +- vendor/github.com/coreos/bbolt/bolt_unix.go | 7 +- .../github.com/coreos/bbolt/bolt_unix_aix.go | 90 +++++++++ .../coreos/bbolt/bolt_unix_solaris.go | 5 +- .../github.com/coreos/bbolt/bolt_windows.go | 34 ++-- .../github.com/coreos/bbolt/boltsync_unix.go | 2 +- vendor/github.com/coreos/bbolt/bucket.go | 4 +- vendor/github.com/coreos/bbolt/cursor.go | 4 +- vendor/github.com/coreos/bbolt/db.go | 60 ++++-- vendor/github.com/coreos/bbolt/doc.go | 4 +- vendor/github.com/coreos/bbolt/errors.go | 2 +- vendor/github.com/coreos/bbolt/freelist.go | 131 +++++++++---- .../github.com/coreos/bbolt/freelist_hmap.go | 178 ++++++++++++++++++ vendor/github.com/coreos/bbolt/go.mod | 3 + vendor/github.com/coreos/bbolt/node.go | 2 +- vendor/github.com/coreos/bbolt/page.go | 2 +- vendor/github.com/coreos/bbolt/tx.go | 29 ++- 65 files changed, 653 insertions(+), 228 deletions(-) create mode 100644 vendor/github.com/coreos/bbolt/bolt_riscv64.go create mode 100644 vendor/github.com/coreos/bbolt/bolt_unix_aix.go create mode 100644 vendor/github.com/coreos/bbolt/freelist_hmap.go create mode 100644 vendor/github.com/coreos/bbolt/go.mod diff --git a/libbeat/cmd/instance/metrics.go b/libbeat/cmd/instance/metrics.go index 54cd3ab55..36c1016d3 100644 --- a/libbeat/cmd/instance/metrics.go +++ b/libbeat/cmd/instance/metrics.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin,cgo freebsd,cgo linux windows +// +build aix darwin,cgo freebsd,cgo linux windows package instance diff --git a/libbeat/cmd/instance/metrics_other.go b/libbeat/cmd/instance/metrics_other.go index 2ea96cd6c..dd6cbc872 100644 --- a/libbeat/cmd/instance/metrics_other.go +++ b/libbeat/cmd/instance/metrics_other.go @@ -17,7 +17,7 @@ // +build !darwin !cgo // +build !freebsd !cgo -// +build !linux,!windows +// +build !aix,!linux,!windows package instance diff --git a/libbeat/metric/system/cpu/cpu.go b/libbeat/metric/system/cpu/cpu.go index abf274ff0..518b8d97e 100644 --- a/libbeat/metric/system/cpu/cpu.go +++ b/libbeat/metric/system/cpu/cpu.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux openbsd windows +// +build aix darwin freebsd linux openbsd windows package cpu diff --git a/libbeat/metric/system/cpu/cpu_test.go b/libbeat/metric/system/cpu/cpu_test.go index f45e43136..b4e72a695 100644 --- a/libbeat/metric/system/cpu/cpu_test.go +++ b/libbeat/metric/system/cpu/cpu_test.go @@ -16,7 +16,7 @@ // under the License. // +build !integration -// +build darwin freebsd linux openbsd windows +// +build aix darwin freebsd linux openbsd windows package cpu diff --git a/libbeat/metric/system/memory/memory.go b/libbeat/metric/system/memory/memory.go index 9351ac08e..1ce83f621 100644 --- a/libbeat/metric/system/memory/memory.go +++ b/libbeat/metric/system/memory/memory.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux openbsd windows +// +build aix darwin freebsd linux openbsd windows package memory diff --git a/libbeat/metric/system/memory/memory_test.go b/libbeat/metric/system/memory/memory_test.go index 58fb74021..8134d9a41 100644 --- a/libbeat/metric/system/memory/memory_test.go +++ b/libbeat/metric/system/memory/memory_test.go @@ -16,7 +16,7 @@ // under the License. // +build !integration -// +build darwin freebsd linux openbsd windows +// +build aix darwin freebsd linux openbsd windows package memory diff --git a/libbeat/metric/system/process/process.go b/libbeat/metric/system/process/process.go index 7c5d70ebb..a5db10d7f 100644 --- a/libbeat/metric/system/process/process.go +++ b/libbeat/metric/system/process/process.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux windows +// +build aix darwin freebsd linux windows package process diff --git a/libbeat/metric/system/process/process_test.go b/libbeat/metric/system/process/process_test.go index 527f60307..7f6f5608f 100644 --- a/libbeat/metric/system/process/process_test.go +++ b/libbeat/metric/system/process/process_test.go @@ -16,7 +16,7 @@ // under the License. // +build !integration -// +build darwin freebsd linux windows +// +build aix darwin freebsd linux windows package process diff --git a/metricbeat/module/system/core/core.go b/metricbeat/module/system/core/core.go index 9fdb3178a..070982062 100644 --- a/metricbeat/module/system/core/core.go +++ b/metricbeat/module/system/core/core.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux openbsd windows +// +build aix darwin freebsd linux openbsd windows package core diff --git a/metricbeat/module/system/core/core_test.go b/metricbeat/module/system/core/core_test.go index 3e8a66f7b..285a20863 100644 --- a/metricbeat/module/system/core/core_test.go +++ b/metricbeat/module/system/core/core_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux openbsd windows +// +build aix darwin freebsd linux openbsd windows package core diff --git a/metricbeat/module/system/cpu/cpu.go b/metricbeat/module/system/cpu/cpu.go index 8d017f0d3..ffdceb07a 100644 --- a/metricbeat/module/system/cpu/cpu.go +++ b/metricbeat/module/system/cpu/cpu.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux openbsd windows +// +build aix darwin freebsd linux openbsd windows package cpu diff --git a/metricbeat/module/system/cpu/cpu_test.go b/metricbeat/module/system/cpu/cpu_test.go index e3b7ba664..2fbf99cf9 100644 --- a/metricbeat/module/system/cpu/cpu_test.go +++ b/metricbeat/module/system/cpu/cpu_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux openbsd windows +// +build aix darwin freebsd linux openbsd windows package cpu diff --git a/metricbeat/module/system/filesystem/filesystem.go b/metricbeat/module/system/filesystem/filesystem.go index 765a874cb..cf83d16b9 100644 --- a/metricbeat/module/system/filesystem/filesystem.go +++ b/metricbeat/module/system/filesystem/filesystem.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux openbsd windows +// +build aix darwin freebsd linux openbsd windows package filesystem diff --git a/metricbeat/module/system/filesystem/filesystem_test.go b/metricbeat/module/system/filesystem/filesystem_test.go index 87ea5c072..fef66aa98 100644 --- a/metricbeat/module/system/filesystem/filesystem_test.go +++ b/metricbeat/module/system/filesystem/filesystem_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux openbsd windows +// +build aix darwin freebsd linux openbsd windows package filesystem diff --git a/metricbeat/module/system/filesystem/helper.go b/metricbeat/module/system/filesystem/helper.go index d64ef95f5..4259d4b04 100644 --- a/metricbeat/module/system/filesystem/helper.go +++ b/metricbeat/module/system/filesystem/helper.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux openbsd windows +// +build aix darwin freebsd linux openbsd windows package filesystem diff --git a/metricbeat/module/system/filesystem/helper_test.go b/metricbeat/module/system/filesystem/helper_test.go index 7ee8d997c..d83e95663 100644 --- a/metricbeat/module/system/filesystem/helper_test.go +++ b/metricbeat/module/system/filesystem/helper_test.go @@ -16,7 +16,7 @@ // under the License. // +build !integration -// +build darwin freebsd linux openbsd windows +// +build aix darwin freebsd linux openbsd windows package filesystem diff --git a/metricbeat/module/system/fsstat/fsstat.go b/metricbeat/module/system/fsstat/fsstat.go index a8a1eaa0c..ed6aea828 100644 --- a/metricbeat/module/system/fsstat/fsstat.go +++ b/metricbeat/module/system/fsstat/fsstat.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux openbsd windows +// +build aix darwin freebsd linux openbsd windows package fsstat diff --git a/metricbeat/module/system/fsstat/fsstat_test.go b/metricbeat/module/system/fsstat/fsstat_test.go index df591a108..3cfe6f865 100644 --- a/metricbeat/module/system/fsstat/fsstat_test.go +++ b/metricbeat/module/system/fsstat/fsstat_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux openbsd windows +// +build aix darwin freebsd linux openbsd windows package fsstat diff --git a/metricbeat/module/system/load/load.go b/metricbeat/module/system/load/load.go index 1e7a6fe2b..b19043a2c 100644 --- a/metricbeat/module/system/load/load.go +++ b/metricbeat/module/system/load/load.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux openbsd +// +build aix darwin freebsd linux openbsd package load diff --git a/metricbeat/module/system/load/load_test.go b/metricbeat/module/system/load/load_test.go index 6a8fadcc7..efb48edc1 100644 --- a/metricbeat/module/system/load/load_test.go +++ b/metricbeat/module/system/load/load_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux openbsd +// +build aix darwin freebsd linux openbsd package load diff --git a/metricbeat/module/system/memory/memory.go b/metricbeat/module/system/memory/memory.go index 90283f801..c2a1e6a43 100644 --- a/metricbeat/module/system/memory/memory.go +++ b/metricbeat/module/system/memory/memory.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux openbsd windows +// +build aix darwin freebsd linux openbsd windows package memory diff --git a/metricbeat/module/system/memory/memory_test.go b/metricbeat/module/system/memory/memory_test.go index 978d2de8a..8de6087b7 100644 --- a/metricbeat/module/system/memory/memory_test.go +++ b/metricbeat/module/system/memory/memory_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux openbsd windows +// +build aix darwin freebsd linux openbsd windows package memory diff --git a/metricbeat/module/system/network/network.go b/metricbeat/module/system/network/network.go index d9c04d834..25db69c45 100644 --- a/metricbeat/module/system/network/network.go +++ b/metricbeat/module/system/network/network.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux windows +// +build aix darwin freebsd linux windows package network diff --git a/metricbeat/module/system/network/network_test.go b/metricbeat/module/system/network/network_test.go index 999a1b91c..9a4783b68 100644 --- a/metricbeat/module/system/network/network_test.go +++ b/metricbeat/module/system/network/network_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux windows +// +build aix darwin freebsd linux windows package network diff --git a/metricbeat/module/system/process/process.go b/metricbeat/module/system/process/process.go index 9c7541773..c15fe15a9 100644 --- a/metricbeat/module/system/process/process.go +++ b/metricbeat/module/system/process/process.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux windows +// +build aix darwin freebsd linux windows package process diff --git a/metricbeat/module/system/process/process_test.go b/metricbeat/module/system/process/process_test.go index c997410a4..b1fbe5de7 100644 --- a/metricbeat/module/system/process/process_test.go +++ b/metricbeat/module/system/process/process_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux windows +// +build aix darwin freebsd linux windows package process diff --git a/metricbeat/module/system/process_summary/process_summary.go b/metricbeat/module/system/process_summary/process_summary.go index 071a7b2a6..f7c5b0e9e 100644 --- a/metricbeat/module/system/process_summary/process_summary.go +++ b/metricbeat/module/system/process_summary/process_summary.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux windows +// +build aix darwin freebsd linux windows package process_summary diff --git a/metricbeat/module/system/process_summary/process_summary_test.go b/metricbeat/module/system/process_summary/process_summary_test.go index 55afce8f1..6879a8b54 100644 --- a/metricbeat/module/system/process_summary/process_summary_test.go +++ b/metricbeat/module/system/process_summary/process_summary_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin freebsd linux windows +// +build aix darwin freebsd linux windows package process_summary diff --git a/metricbeat/module/system/test_system.py b/metricbeat/module/system/test_system.py index cc6b731b1..d8c234a11 100644 --- a/metricbeat/module/system/test_system.py +++ b/metricbeat/module/system/test_system.py @@ -56,7 +56,7 @@ SYSTEM_PROCESS_FIELDS = ["cpu", "memory", "state"] class Test(metricbeat.BaseTest): - @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd|aix", sys.platform), "os") def test_cpu(self): """ Test cpu system output. @@ -79,7 +79,7 @@ class Test(metricbeat.BaseTest): cpu = evt["system"]["cpu"] self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS), cpu.keys()) - @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd|aix", sys.platform), "os") def test_cpu_ticks_option(self): """ Test cpu_ticks configuration option. @@ -105,7 +105,7 @@ class Test(metricbeat.BaseTest): cpuStats = evt["system"]["cpu"] self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS_ALL), cpuStats.keys()) - @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd|aix", sys.platform), "os") def test_core(self): """ Test core system output. @@ -128,7 +128,7 @@ class Test(metricbeat.BaseTest): core = evt["system"]["core"] self.assertCountEqual(self.de_dot(SYSTEM_CORE_FIELDS), core.keys()) - @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd|aix", sys.platform), "os") def test_core_with_cpu_ticks(self): """ Test core system output. @@ -154,7 +154,7 @@ class Test(metricbeat.BaseTest): core = evt["system"]["core"] self.assertCountEqual(self.de_dot(SYSTEM_CORE_FIELDS_ALL), core.keys()) - @unittest.skipUnless(re.match("(?i)linux|darwin|freebsd|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)linux|darwin|freebsd|openbsd|aix", sys.platform), "os") def test_load(self): """ Test system load. @@ -224,7 +224,7 @@ class Test(metricbeat.BaseTest): diskio = evt["system"]["diskio"] self.assertCountEqual(self.de_dot(SYSTEM_DISKIO_FIELDS_LINUX), diskio.keys()) - @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd|aix", sys.platform), "os") def test_filesystem(self): """ Test system/filesystem output. @@ -247,7 +247,7 @@ class Test(metricbeat.BaseTest): filesystem = evt["system"]["filesystem"] self.assertCountEqual(self.de_dot(SYSTEM_FILESYSTEM_FIELDS), filesystem.keys()) - @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd|aix", sys.platform), "os") def test_fsstat(self): """ Test system/fsstat output. @@ -270,7 +270,7 @@ class Test(metricbeat.BaseTest): fsstat = evt["system"]["fsstat"] self.assertCountEqual(SYSTEM_FSSTAT_FIELDS, fsstat.keys()) - @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd|aix", sys.platform), "os") def test_memory(self): """ Test system memory output. @@ -310,7 +310,7 @@ class Test(metricbeat.BaseTest): used_p = float(swap["used"]["bytes"]) / swap["total"] self.assertAlmostEqual(swap["used"]["pct"], used_p, places=4) - @unittest.skipUnless(re.match("(?i)darwin|win|linux|freebsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)darwin|win|linux|freebsd|aix", sys.platform), "os") def test_network(self): """ Test system/network output. @@ -333,7 +333,7 @@ class Test(metricbeat.BaseTest): network = evt["system"]["network"] self.assertCountEqual(self.de_dot(SYSTEM_NETWORK_FIELDS), network.keys()) - @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|aix", sys.platform), "os") def test_process_summary(self): """ Test system/process_summary output. @@ -366,7 +366,7 @@ class Test(metricbeat.BaseTest): assert summary["total"] == summary["sleeping"] + summary["running"] + \ summary["idle"] + summary["stopped"] + summary["zombie"] + summary["unknown"] - @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|aix", sys.platform), "os") def test_process(self): """ Test system/process output. @@ -428,7 +428,7 @@ class Test(metricbeat.BaseTest): or sys.platform.startswith("darwin"): self.assertTrue(found_env, "env not found in any process events") - @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|aix", sys.platform), "os") def test_process_metricbeat(self): """ Checks that the per proc stats are found in the output and @@ -453,7 +453,7 @@ class Test(metricbeat.BaseTest): assert isinstance(output["system.process.cpu.start_time"], six.string_types) self.check_username(output["user.name"]) - @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|aix", sys.platform), "os") def test_socket_summary(self): """ Test system/socket_summary output. diff --git a/metricbeat/module/system/uptime/uptime.go b/metricbeat/module/system/uptime/uptime.go index cbefa8bd4..60caaad8d 100644 --- a/metricbeat/module/system/uptime/uptime.go +++ b/metricbeat/module/system/uptime/uptime.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin linux openbsd windows freebsd,cgo +// +build aix darwin linux openbsd windows freebsd,cgo package uptime diff --git a/metricbeat/module/system/uptime/uptime_test.go b/metricbeat/module/system/uptime/uptime_test.go index d0542c50d..8d40f64db 100644 --- a/metricbeat/module/system/uptime/uptime_test.go +++ b/metricbeat/module/system/uptime/uptime_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -// +build darwin linux openbsd windows freebsd,cgo +// +build aix darwin linux openbsd windows freebsd,cgo package uptime diff --git a/metricbeat/tests/system/test_base.py b/metricbeat/tests/system/test_base.py index 5e52872bb..3495e4f2a 100644 --- a/metricbeat/tests/system/test_base.py +++ b/metricbeat/tests/system/test_base.py @@ -12,7 +12,7 @@ class Test(BaseTest): COMPOSE_SERVICES = ['elasticsearch', 'kibana'] - @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd|aix", sys.platform), "os") def test_start_stop(self): """ Metricbeat starts and stops without error. diff --git a/metricbeat/tests/system/test_processors.py b/metricbeat/tests/system/test_processors.py index 2f7d131d1..13de2eeaf 100644 --- a/metricbeat/tests/system/test_processors.py +++ b/metricbeat/tests/system/test_processors.py @@ -4,7 +4,7 @@ import metricbeat import unittest -@unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd", sys.platform), "os") +@unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|aix", sys.platform), "os") class Test(metricbeat.BaseTest): def test_drop_fields(self): diff --git a/metricbeat/tests/system/test_reload.py b/metricbeat/tests/system/test_reload.py index 29d82bbf8..74d396ad0 100644 --- a/metricbeat/tests/system/test_reload.py +++ b/metricbeat/tests/system/test_reload.py @@ -16,7 +16,7 @@ import time class Test(metricbeat.BaseTest): - @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd|aix", sys.platform), "os") def test_reload(self): """ Test basic reload @@ -42,7 +42,7 @@ class Test(metricbeat.BaseTest): self.wait_until(lambda: self.output_lines() > 0) proc.check_kill_and_wait() - @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd|aix", sys.platform), "os") def test_start_stop(self): """ Test if module is properly started and stopped @@ -88,7 +88,7 @@ class Test(metricbeat.BaseTest): proc.check_kill_and_wait() - @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd|aix", sys.platform), "os") def test_wrong_module_no_reload(self): """ Test beat errors when reload is disabled and some module config is wrong diff --git a/vendor/github.com/coreos/bbolt/Makefile b/vendor/github.com/coreos/bbolt/Makefile index 43b94f3bd..2968aaa61 100644 --- a/vendor/github.com/coreos/bbolt/Makefile +++ b/vendor/github.com/coreos/bbolt/Makefile @@ -5,7 +5,9 @@ GOLDFLAGS="-X main.branch $(BRANCH) -X main.commit $(COMMIT)" default: build race: - @go test -v -race -test.run="TestSimulate_(100op|1000op)" + @TEST_FREELIST_TYPE=hashmap go test -v -race -test.run="TestSimulate_(100op|1000op)" + @echo "array freelist test" + @TEST_FREELIST_TYPE=array go test -v -race -test.run="TestSimulate_(100op|1000op)" fmt: !(gofmt -l -s -d $(shell find . -name \*.go) | grep '[a-z]') @@ -20,11 +22,17 @@ unused: # go get github.com/kisielk/errcheck errcheck: - @errcheck -ignorepkg=bytes -ignore=os:Remove github.com/coreos/bbolt + @errcheck -ignorepkg=bytes -ignore=os:Remove go.etcd.io/bbolt test: - go test -timeout 20m -v -coverprofile cover.out -covermode atomic + TEST_FREELIST_TYPE=hashmap go test -timeout 20m -v -coverprofile cover.out -covermode atomic # Note: gets "program not an importable package" in out of path builds - go test -v ./cmd/bolt + TEST_FREELIST_TYPE=hashmap go test -v ./cmd/bbolt + + @echo "array freelist test" + + @TEST_FREELIST_TYPE=array go test -timeout 20m -v -coverprofile cover.out -covermode atomic + # Note: gets "program not an importable package" in out of path builds + @TEST_FREELIST_TYPE=array go test -v ./cmd/bbolt .PHONY: race fmt errcheck test gosimple unused diff --git a/vendor/github.com/coreos/bbolt/README.md b/vendor/github.com/coreos/bbolt/README.md index 015f0efbe..2dff3761d 100644 --- a/vendor/github.com/coreos/bbolt/README.md +++ b/vendor/github.com/coreos/bbolt/README.md @@ -1,9 +1,12 @@ bbolt -==== +===== -[![Go Report Card](https://goreportcard.com/badge/github.com/coreos/bbolt?style=flat-square)](https://goreportcard.com/report/github.com/coreos/bbolt) -[![Coverage](https://codecov.io/gh/coreos/bbolt/branch/master/graph/badge.svg)](https://codecov.io/gh/coreos/bbolt) -[![Godoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/coreos/bbolt) +[![Go Report Card](https://goreportcard.com/badge/github.com/etcd-io/bbolt?style=flat-square)](https://goreportcard.com/report/github.com/etcd-io/bbolt) +[![Coverage](https://codecov.io/gh/etcd-io/bbolt/branch/master/graph/badge.svg)](https://codecov.io/gh/etcd-io/bbolt) +[![Build Status Travis](https://img.shields.io/travis/etcd-io/bboltlabs.svg?style=flat-square&&branch=master)](https://travis-ci.com/etcd-io/bbolt) +[![Godoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/etcd-io/bbolt) +[![Releases](https://img.shields.io/github/release/etcd-io/bbolt/all.svg?style=flat-square)](https://github.com/etcd-io/bbolt/releases) +[![LICENSE](https://img.shields.io/github/license/etcd-io/bbolt.svg?style=flat-square)](https://github.com/etcd-io/bbolt/blob/master/LICENSE) bbolt is a fork of [Ben Johnson's][gh_ben] [Bolt][bolt] key/value store. The purpose of this fork is to provide the Go community with an active @@ -33,36 +36,42 @@ consistency and thread safety. Bolt is currently used in high-load production environments serving databases as large as 1TB. Many companies such as Shopify and Heroku use Bolt-backed services every day. +## Project versioning + +bbolt uses [semantic versioning](http://semver.org). +API should not change between patch and minor releases. +New minor versions may add additional features to the API. + ## Table of Contents -- [Getting Started](#getting-started) - - [Installing](#installing) - - [Opening a database](#opening-a-database) - - [Transactions](#transactions) - - [Read-write transactions](#read-write-transactions) - - [Read-only transactions](#read-only-transactions) - - [Batch read-write transactions](#batch-read-write-transactions) - - [Managing transactions manually](#managing-transactions-manually) - - [Using buckets](#using-buckets) - - [Using key/value pairs](#using-keyvalue-pairs) - - [Autoincrementing integer for the bucket](#autoincrementing-integer-for-the-bucket) - - [Iterating over keys](#iterating-over-keys) - - [Prefix scans](#prefix-scans) - - [Range scans](#range-scans) - - [ForEach()](#foreach) - - [Nested buckets](#nested-buckets) - - [Database backups](#database-backups) - - [Statistics](#statistics) - - [Read-Only Mode](#read-only-mode) - - [Mobile Use (iOS/Android)](#mobile-use-iosandroid) -- [Resources](#resources) -- [Comparison with other databases](#comparison-with-other-databases) - - [Postgres, MySQL, & other relational databases](#postgres-mysql--other-relational-databases) - - [LevelDB, RocksDB](#leveldb-rocksdb) - - [LMDB](#lmdb) -- [Caveats & Limitations](#caveats--limitations) -- [Reading the Source](#reading-the-source) -- [Other Projects Using Bolt](#other-projects-using-bolt) + - [Getting Started](#getting-started) + - [Installing](#installing) + - [Opening a database](#opening-a-database) + - [Transactions](#transactions) + - [Read-write transactions](#read-write-transactions) + - [Read-only transactions](#read-only-transactions) + - [Batch read-write transactions](#batch-read-write-transactions) + - [Managing transactions manually](#managing-transactions-manually) + - [Using buckets](#using-buckets) + - [Using key/value pairs](#using-keyvalue-pairs) + - [Autoincrementing integer for the bucket](#autoincrementing-integer-for-the-bucket) + - [Iterating over keys](#iterating-over-keys) + - [Prefix scans](#prefix-scans) + - [Range scans](#range-scans) + - [ForEach()](#foreach) + - [Nested buckets](#nested-buckets) + - [Database backups](#database-backups) + - [Statistics](#statistics) + - [Read-Only Mode](#read-only-mode) + - [Mobile Use (iOS/Android)](#mobile-use-iosandroid) + - [Resources](#resources) + - [Comparison with other databases](#comparison-with-other-databases) + - [Postgres, MySQL, & other relational databases](#postgres-mysql--other-relational-databases) + - [LevelDB, RocksDB](#leveldb-rocksdb) + - [LMDB](#lmdb) + - [Caveats & Limitations](#caveats--limitations) + - [Reading the Source](#reading-the-source) + - [Other Projects Using Bolt](#other-projects-using-bolt) ## Getting Started @@ -71,13 +80,28 @@ Shopify and Heroku use Bolt-backed services every day. To start using Bolt, install Go and run `go get`: ```sh -$ go get github.com/coreos/bbolt/... +$ go get go.etcd.io/bbolt/... ``` This will retrieve the library and install the `bolt` command line utility into your `$GOBIN` path. +### Importing bbolt + +To use bbolt as an embedded key-value store, import as: + +```go +import bolt "go.etcd.io/bbolt" + +db, err := bolt.Open(path, 0666, nil) +if err != nil { + return err +} +defer db.Close() +``` + + ### Opening a database The top-level object in Bolt is a `DB`. It is represented as a single file on @@ -91,7 +115,7 @@ package main import ( "log" - bolt "github.com/coreos/bbolt" + bolt "go.etcd.io/bbolt" ) func main() { @@ -251,7 +275,7 @@ should be writable. ### Using buckets Buckets are collections of key/value pairs within the database. All keys in a -bucket must be unique. You can create a bucket using the `DB.CreateBucket()` +bucket must be unique. You can create a bucket using the `Tx.CreateBucket()` function: ```go @@ -534,7 +558,7 @@ this from a read-only transaction, it will perform a hot backup and not block your other database reads and writes. By default, it will use a regular file handle which will utilize the operating -system's page cache. See the [`Tx`](https://godoc.org/github.com/coreos/bbolt#Tx) +system's page cache. See the [`Tx`](https://godoc.org/go.etcd.io/bbolt#Tx) documentation for information about optimizing for larger-than-RAM datasets. One common use case is to backup over HTTP so you can use tools like `cURL` to @@ -875,54 +899,58 @@ them via pull request. Below is a list of public, open source projects that use Bolt: -* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files. -* [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard. +* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend. * [Bazil](https://bazil.org/) - A file system that lets your data reside where it is most convenient for it to reside. -* [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb. -* [Skybox Analytics](https://github.com/skybox/skybox) - A standalone funnel analysis tool for web analytics. -* [Scuttlebutt](https://github.com/benbjohnson/scuttlebutt) - Uses Bolt to store and process all Twitter mentions of GitHub projects. -* [Wiki](https://github.com/peterhellberg/wiki) - A tiny wiki using Goji, BoltDB and Blackfriday. +* [bolter](https://github.com/hasit/bolter) - Command-line app for viewing BoltDB file in your terminal. +* [boltcli](https://github.com/spacewander/boltcli) - the redis-cli for boltdb with Lua script support. +* [BoltHold](https://github.com/timshannon/bolthold) - An embeddable NoSQL store for Go types built on BoltDB +* [BoltStore](https://github.com/yosssi/boltstore) - Session store using Bolt. +* [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners. +* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files. +* [bleve](http://www.blevesearch.com/) - A pure Go search engine similar to ElasticSearch that uses Bolt as the default storage backend. +* [btcwallet](https://github.com/btcsuite/btcwallet) - A bitcoin wallet. +* [buckets](https://github.com/joyrexus/buckets) - a bolt wrapper streamlining + simple tx and key scans. +* [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend. * [ChainStore](https://github.com/pressly/chainstore) - Simple key-value interface to a variety of storage engines organized as a chain of operations. -* [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite. -* [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin". +* [Consul](https://github.com/hashicorp/consul) - Consul is service discovery and configuration made easy. Distributed, highly available, and datacenter-aware. +* [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb. +* [dcrwallet](https://github.com/decred/dcrwallet) - A wallet for the Decred cryptocurrency. +* [drive](https://github.com/odeke-em/drive) - drive is an unofficial Google Drive command line client for \*NIX operating systems. * [event-shuttle](https://github.com/sclasen/event-shuttle) - A Unix system service to collect and reliably deliver messages to Kafka. +* [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data. +* [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service. +* [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB. +* [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter. +* [gopherpit](https://github.com/gopherpit/gopherpit) - A web service to manage Go remote import paths with custom domains +* [gokv](https://github.com/philippgille/gokv) - Simple key-value store abstraction and implementations for Go (Redis, Consul, etcd, bbolt, BadgerDB, LevelDB, Memcached, DynamoDB, S3, PostgreSQL, MongoDB, CockroachDB and many more) +* [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin". +* [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics. +* [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters. * [ipxed](https://github.com/kelseyhightower/ipxed) - Web interface and api for ipxed. -* [BoltStore](https://github.com/yosssi/boltstore) - Session store using Bolt. -* [photosite/session](https://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site. +* [Ironsmith](https://github.com/timshannon/ironsmith) - A simple, script-driven continuous integration (build - > test -> release) tool, with no external dependencies +* [Kala](https://github.com/ajvb/kala) - Kala is a modern job scheduler optimized to run on a single node. It is persistent, JSON over HTTP API, ISO 8601 duration notation, and dependent jobs. +* [Key Value Access Langusge (KVAL)](https://github.com/kval-access-language) - A proposed grammar for key-value datastores offering a bbolt binding. * [LedisDB](https://github.com/siddontang/ledisdb) - A high performance NoSQL, using Bolt as optional storage. -* [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters. -* [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend. -* [bleve](http://www.blevesearch.com/) - A pure Go search engine similar to ElasticSearch that uses Bolt as the default storage backend. -* [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server. -* [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read. -* [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics. -* [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data. +* [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores. +* [mbuckets](https://github.com/abhigupta912/mbuckets) - A Bolt wrapper that allows easy operations on multi level (nested) buckets. +* [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite. +* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files. +* [NATS](https://github.com/nats-io/nats-streaming-server) - NATS Streaming uses bbolt for message and metadata storage. +* [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard. +* [photosite/session](https://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site. * [Prometheus Annotation Server](https://github.com/oliver006/prom_annotation_server) - Annotation server for PromDash & Prometheus service monitoring system. -* [Consul](https://github.com/hashicorp/consul) - Consul is service discovery and configuration made easy. Distributed, highly available, and datacenter-aware. -* [Kala](https://github.com/ajvb/kala) - Kala is a modern job scheduler optimized to run on a single node. It is persistent, JSON over HTTP API, ISO 8601 duration notation, and dependent jobs. -* [drive](https://github.com/odeke-em/drive) - drive is an unofficial Google Drive command line client for \*NIX operating systems. +* [reef-pi](https://github.com/reef-pi/reef-pi) - reef-pi is an award winning, modular, DIY reef tank controller using easy to learn electronics based on a Raspberry Pi. +* [Request Baskets](https://github.com/darklynx/request-baskets) - A web service to collect arbitrary HTTP requests and inspect them via REST API or simple web UI, similar to [RequestBin](http://requestb.in/) service +* [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read. * [stow](https://github.com/djherbis/stow) - a persistence manager for objects backed by boltdb. -* [buckets](https://github.com/joyrexus/buckets) - a bolt wrapper streamlining - simple tx and key scans. -* [mbuckets](https://github.com/abhigupta912/mbuckets) - A Bolt wrapper that allows easy operations on multi level (nested) buckets. -* [Request Baskets](https://github.com/darklynx/request-baskets) - A web service to collect arbitrary HTTP requests and inspect them via REST API or simple web UI, similar to [RequestBin](http://requestb.in/) service -* [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service. -* [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners. -* [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores. * [Storm](https://github.com/asdine/storm) - Simple and powerful ORM for BoltDB. -* [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB. * [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings. -* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend. -* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files. -* [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter. +* [Skybox Analytics](https://github.com/skybox/skybox) - A standalone funnel analysis tool for web analytics. +* [Scuttlebutt](https://github.com/benbjohnson/scuttlebutt) - Uses Bolt to store and process all Twitter mentions of GitHub projects. +* [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server. * [torrent](https://github.com/anacrolix/torrent) - Full-featured BitTorrent client package and utilities in Go. BoltDB is a storage backend in development. -* [gopherpit](https://github.com/gopherpit/gopherpit) - A web service to manage Go remote import paths with custom domains -* [bolter](https://github.com/hasit/bolter) - Command-line app for viewing BoltDB file in your terminal. -* [boltcli](https://github.com/spacewander/boltcli) - the redis-cli for boltdb with Lua script support. -* [btcwallet](https://github.com/btcsuite/btcwallet) - A bitcoin wallet. -* [dcrwallet](https://github.com/decred/dcrwallet) - A wallet for the Decred cryptocurrency. -* [Ironsmith](https://github.com/timshannon/ironsmith) - A simple, script-driven continuous integration (build - > test -> release) tool, with no external dependencies -* [BoltHold](https://github.com/timshannon/bolthold) - An embeddable NoSQL store for Go types built on BoltDB +* [Wiki](https://github.com/peterhellberg/wiki) - A tiny wiki using Goji, BoltDB and Blackfriday. If you are using Bolt in a project please send a pull request to add it to the list. diff --git a/vendor/github.com/coreos/bbolt/bolt_386.go b/vendor/github.com/coreos/bbolt/bolt_386.go index 820d533c1..4d35ee7cf 100644 --- a/vendor/github.com/coreos/bbolt/bolt_386.go +++ b/vendor/github.com/coreos/bbolt/bolt_386.go @@ -1,4 +1,4 @@ -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0x7FFFFFFF // 2GB diff --git a/vendor/github.com/coreos/bbolt/bolt_amd64.go b/vendor/github.com/coreos/bbolt/bolt_amd64.go index 98fafdb47..60a52dad5 100644 --- a/vendor/github.com/coreos/bbolt/bolt_amd64.go +++ b/vendor/github.com/coreos/bbolt/bolt_amd64.go @@ -1,4 +1,4 @@ -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/coreos/bbolt/bolt_arm.go b/vendor/github.com/coreos/bbolt/bolt_arm.go index 7e5cb4b94..105d27ddb 100644 --- a/vendor/github.com/coreos/bbolt/bolt_arm.go +++ b/vendor/github.com/coreos/bbolt/bolt_arm.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import "unsafe" diff --git a/vendor/github.com/coreos/bbolt/bolt_arm64.go b/vendor/github.com/coreos/bbolt/bolt_arm64.go index b26d84f91..f5aa2a5ee 100644 --- a/vendor/github.com/coreos/bbolt/bolt_arm64.go +++ b/vendor/github.com/coreos/bbolt/bolt_arm64.go @@ -1,6 +1,6 @@ // +build arm64 -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/coreos/bbolt/bolt_linux.go b/vendor/github.com/coreos/bbolt/bolt_linux.go index 2b6766614..7707bcacf 100644 --- a/vendor/github.com/coreos/bbolt/bolt_linux.go +++ b/vendor/github.com/coreos/bbolt/bolt_linux.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "syscall" diff --git a/vendor/github.com/coreos/bbolt/bolt_mips64x.go b/vendor/github.com/coreos/bbolt/bolt_mips64x.go index 134b578bd..baeb289fd 100644 --- a/vendor/github.com/coreos/bbolt/bolt_mips64x.go +++ b/vendor/github.com/coreos/bbolt/bolt_mips64x.go @@ -1,6 +1,6 @@ // +build mips64 mips64le -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0x8000000000 // 512GB diff --git a/vendor/github.com/coreos/bbolt/bolt_mipsx.go b/vendor/github.com/coreos/bbolt/bolt_mipsx.go index d5ecb0597..2d9b1a91f 100644 --- a/vendor/github.com/coreos/bbolt/bolt_mipsx.go +++ b/vendor/github.com/coreos/bbolt/bolt_mipsx.go @@ -1,6 +1,6 @@ // +build mips mipsle -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0x40000000 // 1GB diff --git a/vendor/github.com/coreos/bbolt/bolt_openbsd.go b/vendor/github.com/coreos/bbolt/bolt_openbsd.go index 7058c3d73..d7f50358e 100644 --- a/vendor/github.com/coreos/bbolt/bolt_openbsd.go +++ b/vendor/github.com/coreos/bbolt/bolt_openbsd.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "syscall" diff --git a/vendor/github.com/coreos/bbolt/bolt_ppc.go b/vendor/github.com/coreos/bbolt/bolt_ppc.go index 55cb8a72c..69804714a 100644 --- a/vendor/github.com/coreos/bbolt/bolt_ppc.go +++ b/vendor/github.com/coreos/bbolt/bolt_ppc.go @@ -1,6 +1,6 @@ // +build ppc -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0x7FFFFFFF // 2GB diff --git a/vendor/github.com/coreos/bbolt/bolt_ppc64.go b/vendor/github.com/coreos/bbolt/bolt_ppc64.go index 9331d9771..356590857 100644 --- a/vendor/github.com/coreos/bbolt/bolt_ppc64.go +++ b/vendor/github.com/coreos/bbolt/bolt_ppc64.go @@ -1,6 +1,6 @@ // +build ppc64 -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/coreos/bbolt/bolt_ppc64le.go b/vendor/github.com/coreos/bbolt/bolt_ppc64le.go index 8c143bc5d..422c7c69d 100644 --- a/vendor/github.com/coreos/bbolt/bolt_ppc64le.go +++ b/vendor/github.com/coreos/bbolt/bolt_ppc64le.go @@ -1,6 +1,6 @@ // +build ppc64le -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/coreos/bbolt/bolt_riscv64.go b/vendor/github.com/coreos/bbolt/bolt_riscv64.go new file mode 100644 index 000000000..07b4b47cd --- /dev/null +++ b/vendor/github.com/coreos/bbolt/bolt_riscv64.go @@ -0,0 +1,12 @@ +// +build riscv64 + +package bbolt + +// maxMapSize represents the largest mmap size supported by Bolt. +const maxMapSize = 0xFFFFFFFFFFFF // 256TB + +// maxAllocSize is the size used when creating array pointers. +const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = true diff --git a/vendor/github.com/coreos/bbolt/bolt_s390x.go b/vendor/github.com/coreos/bbolt/bolt_s390x.go index d7c39af92..6d3fcb825 100644 --- a/vendor/github.com/coreos/bbolt/bolt_s390x.go +++ b/vendor/github.com/coreos/bbolt/bolt_s390x.go @@ -1,6 +1,6 @@ // +build s390x -package bolt +package bbolt // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0xFFFFFFFFFFFF // 256TB diff --git a/vendor/github.com/coreos/bbolt/bolt_unix.go b/vendor/github.com/coreos/bbolt/bolt_unix.go index add3bd882..2938fed58 100644 --- a/vendor/github.com/coreos/bbolt/bolt_unix.go +++ b/vendor/github.com/coreos/bbolt/bolt_unix.go @@ -1,17 +1,16 @@ -// +build !windows,!plan9,!solaris +// +build !windows,!plan9,!solaris,!aix -package bolt +package bbolt import ( "fmt" - "os" "syscall" "time" "unsafe" ) // flock acquires an advisory lock on a file descriptor. -func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error { +func flock(db *DB, exclusive bool, timeout time.Duration) error { var t time.Time if timeout != 0 { t = time.Now() diff --git a/vendor/github.com/coreos/bbolt/bolt_unix_aix.go b/vendor/github.com/coreos/bbolt/bolt_unix_aix.go new file mode 100644 index 000000000..a64c16f51 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/bolt_unix_aix.go @@ -0,0 +1,90 @@ +// +build aix + +package bbolt + +import ( + "fmt" + "syscall" + "time" + "unsafe" + + "golang.org/x/sys/unix" +) + +// flock acquires an advisory lock on a file descriptor. +func flock(db *DB, exclusive bool, timeout time.Duration) error { + var t time.Time + if timeout != 0 { + t = time.Now() + } + fd := db.file.Fd() + var lockType int16 + if exclusive { + lockType = syscall.F_WRLCK + } else { + lockType = syscall.F_RDLCK + } + for { + // Attempt to obtain an exclusive lock. + lock := syscall.Flock_t{Type: lockType} + err := syscall.FcntlFlock(fd, syscall.F_SETLK, &lock) + if err == nil { + return nil + } else if err != syscall.EAGAIN { + return err + } + + // If we timed out then return an error. + if timeout != 0 && time.Since(t) > timeout-flockRetryTimeout { + return ErrTimeout + } + + // Wait for a bit and try again. + time.Sleep(flockRetryTimeout) + } +} + +// funlock releases an advisory lock on a file descriptor. +func funlock(db *DB) error { + var lock syscall.Flock_t + lock.Start = 0 + lock.Len = 0 + lock.Type = syscall.F_UNLCK + lock.Whence = 0 + return syscall.FcntlFlock(uintptr(db.file.Fd()), syscall.F_SETLK, &lock) +} + +// mmap memory maps a DB's data file. +func mmap(db *DB, sz int) error { + // Map the data file to memory. + b, err := unix.Mmap(int(db.file.Fd()), 0, sz, syscall.PROT_READ, syscall.MAP_SHARED|db.MmapFlags) + if err != nil { + return err + } + + // Advise the kernel that the mmap is accessed randomly. + if err := unix.Madvise(b, syscall.MADV_RANDOM); err != nil { + return fmt.Errorf("madvise: %s", err) + } + + // Save the original byte slice and convert to a byte array pointer. + db.dataref = b + db.data = (*[maxMapSize]byte)(unsafe.Pointer(&b[0])) + db.datasz = sz + return nil +} + +// munmap unmaps a DB's data file from memory. +func munmap(db *DB) error { + // Ignore the unmap if we have no mapped data. + if db.dataref == nil { + return nil + } + + // Unmap using the original byte slice. + err := unix.Munmap(db.dataref) + db.dataref = nil + db.data = nil + db.datasz = 0 + return err +} diff --git a/vendor/github.com/coreos/bbolt/bolt_unix_solaris.go b/vendor/github.com/coreos/bbolt/bolt_unix_solaris.go index fd8335ecc..babad6578 100644 --- a/vendor/github.com/coreos/bbolt/bolt_unix_solaris.go +++ b/vendor/github.com/coreos/bbolt/bolt_unix_solaris.go @@ -1,8 +1,7 @@ -package bolt +package bbolt import ( "fmt" - "os" "syscall" "time" "unsafe" @@ -11,7 +10,7 @@ import ( ) // flock acquires an advisory lock on a file descriptor. -func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error { +func flock(db *DB, exclusive bool, timeout time.Duration) error { var t time.Time if timeout != 0 { t = time.Now() diff --git a/vendor/github.com/coreos/bbolt/bolt_windows.go b/vendor/github.com/coreos/bbolt/bolt_windows.go index ca6f9a11c..fca178bd2 100644 --- a/vendor/github.com/coreos/bbolt/bolt_windows.go +++ b/vendor/github.com/coreos/bbolt/bolt_windows.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "fmt" @@ -16,8 +16,6 @@ var ( ) const ( - lockExt = ".lock" - // see https://msdn.microsoft.com/en-us/library/windows/desktop/aa365203(v=vs.85).aspx flagLockExclusive = 2 flagLockFailImmediately = 1 @@ -48,28 +46,24 @@ func fdatasync(db *DB) error { } // flock acquires an advisory lock on a file descriptor. -func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) error { - // Create a separate lock file on windows because a process - // cannot share an exclusive lock on the same file. This is - // needed during Tx.WriteTo(). - f, err := os.OpenFile(db.path+lockExt, os.O_CREATE, mode) - if err != nil { - return err - } - db.lockfile = f - +func flock(db *DB, exclusive bool, timeout time.Duration) error { var t time.Time if timeout != 0 { t = time.Now() } - fd := f.Fd() var flag uint32 = flagLockFailImmediately if exclusive { flag |= flagLockExclusive } for { - // Attempt to obtain an exclusive lock. - err := lockFileEx(syscall.Handle(fd), flag, 0, 1, 0, &syscall.Overlapped{}) + // Fix for https://github.com/etcd-io/bbolt/issues/121. Use byte-range + // -1..0 as the lock on the database file. + var m1 uint32 = (1 << 32) - 1 // -1 in a uint32 + err := lockFileEx(syscall.Handle(db.file.Fd()), flag, 0, 1, 0, &syscall.Overlapped{ + Offset: m1, + OffsetHigh: m1, + }) + if err == nil { return nil } else if err != errLockViolation { @@ -88,9 +82,11 @@ func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) erro // funlock releases an advisory lock on a file descriptor. func funlock(db *DB) error { - err := unlockFileEx(syscall.Handle(db.lockfile.Fd()), 0, 1, 0, &syscall.Overlapped{}) - db.lockfile.Close() - os.Remove(db.path + lockExt) + var m1 uint32 = (1 << 32) - 1 // -1 in a uint32 + err := unlockFileEx(syscall.Handle(db.file.Fd()), 0, 1, 0, &syscall.Overlapped{ + Offset: m1, + OffsetHigh: m1, + }) return err } diff --git a/vendor/github.com/coreos/bbolt/boltsync_unix.go b/vendor/github.com/coreos/bbolt/boltsync_unix.go index f50442523..9587afefe 100644 --- a/vendor/github.com/coreos/bbolt/boltsync_unix.go +++ b/vendor/github.com/coreos/bbolt/boltsync_unix.go @@ -1,6 +1,6 @@ // +build !windows,!plan9,!linux,!openbsd -package bolt +package bbolt // fdatasync flushes written data to a file descriptor. func fdatasync(db *DB) error { diff --git a/vendor/github.com/coreos/bbolt/bucket.go b/vendor/github.com/coreos/bbolt/bucket.go index 44db88b8a..f88968451 100644 --- a/vendor/github.com/coreos/bbolt/bucket.go +++ b/vendor/github.com/coreos/bbolt/bucket.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "bytes" @@ -206,7 +206,7 @@ func (b *Bucket) CreateBucketIfNotExists(key []byte) (*Bucket, error) { } // DeleteBucket deletes a bucket at the given key. -// Returns an error if the bucket does not exists, or if the key represents a non-bucket value. +// Returns an error if the bucket does not exist, or if the key represents a non-bucket value. func (b *Bucket) DeleteBucket(key []byte) error { if b.tx.db == nil { return ErrTxClosed diff --git a/vendor/github.com/coreos/bbolt/cursor.go b/vendor/github.com/coreos/bbolt/cursor.go index 1bdda63a2..98aeb449a 100644 --- a/vendor/github.com/coreos/bbolt/cursor.go +++ b/vendor/github.com/coreos/bbolt/cursor.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "bytes" @@ -366,7 +366,7 @@ func (c *Cursor) node() *node { } for _, ref := range c.stack[:len(c.stack)-1] { _assert(!n.isLeaf, "expected branch node") - n = n.childAt(int(ref.index)) + n = n.childAt(ref.index) } _assert(n.isLeaf, "expected leaf node") return n diff --git a/vendor/github.com/coreos/bbolt/db.go b/vendor/github.com/coreos/bbolt/db.go index ab97c6014..80b0095cc 100644 --- a/vendor/github.com/coreos/bbolt/db.go +++ b/vendor/github.com/coreos/bbolt/db.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "errors" @@ -43,6 +43,16 @@ var defaultPageSize = os.Getpagesize() // The time elapsed between consecutive file locking attempts. const flockRetryTimeout = 50 * time.Millisecond +// FreelistType is the type of the freelist backend +type FreelistType string + +const ( + // FreelistArrayType indicates backend freelist type is array + FreelistArrayType = FreelistType("array") + // FreelistMapType indicates backend freelist type is hashmap + FreelistMapType = FreelistType("hashmap") +) + // DB represents a collection of buckets persisted to a file on disk. // All data access is performed through transactions which can be obtained through the DB. // All the functions on DB will return a ErrDatabaseNotOpen if accessed before Open() is called. @@ -70,6 +80,13 @@ type DB struct { // re-sync during recovery. NoFreelistSync bool + // FreelistType sets the backend freelist type. There are two options. Array which is simple but endures + // dramatic performance degradation if database is large and framentation in freelist is common. + // The alternative one is using hashmap, it is faster in almost all circumstances + // but it doesn't guarantee that it offers the smallest page id available. In normal case it is safe. + // The default type is array + FreelistType FreelistType + // When true, skips the truncate call when growing the database. // Setting this to true is only safe on non-ext3/ext4 systems. // Skipping truncation avoids preallocation of hard drive space and @@ -104,9 +121,9 @@ type DB struct { AllocSize int path string + openFile func(string, int, os.FileMode) (*os.File, error) file *os.File - lockfile *os.File // windows only - dataref []byte // mmap'ed readonly, write throws SEGV + dataref []byte // mmap'ed readonly, write throws SEGV data *[maxMapSize]byte datasz int filesz int // current on disk file size @@ -170,6 +187,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { db.NoGrowSync = options.NoGrowSync db.MmapFlags = options.MmapFlags db.NoFreelistSync = options.NoFreelistSync + db.FreelistType = options.FreelistType // Set default values for later DB operations. db.MaxBatchSize = DefaultMaxBatchSize @@ -182,13 +200,18 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { db.readOnly = true } + db.openFile = options.OpenFile + if db.openFile == nil { + db.openFile = os.OpenFile + } + // Open data file and separate sync handler for metadata writes. - db.path = path var err error - if db.file, err = os.OpenFile(db.path, flag|os.O_CREATE, mode); err != nil { + if db.file, err = db.openFile(path, flag|os.O_CREATE, mode); err != nil { _ = db.close() return nil, err } + db.path = db.file.Name() // Lock file so that other processes using Bolt in read-write mode cannot // use the database at the same time. This would cause corruption since @@ -197,8 +220,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { // if !options.ReadOnly. // The database file is locked using the shared lock (more than one process may // hold a lock at the same time) otherwise (options.ReadOnly is set). - if err := flock(db, mode, !db.readOnly, options.Timeout); err != nil { - db.lockfile = nil // make 'unused' happy. TODO: rework locks + if err := flock(db, !db.readOnly, options.Timeout); err != nil { _ = db.close() return nil, err } @@ -285,7 +307,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) { // concurrent accesses being made to the freelist. func (db *DB) loadFreelist() { db.freelistLoad.Do(func() { - db.freelist = newFreelist() + db.freelist = newFreelist(db.FreelistType) if !db.hasSyncedFreelist() { // Reconstruct free list by scanning the DB. db.freelist.readIDs(db.freepages()) @@ -293,7 +315,7 @@ func (db *DB) loadFreelist() { // Read free list from freelist page. db.freelist.read(db.page(db.meta().freelist)) } - db.stats.FreePageN = len(db.freelist.ids) + db.stats.FreePageN = db.freelist.free_count() }) } @@ -454,8 +476,8 @@ func (db *DB) Close() error { db.metalock.Lock() defer db.metalock.Unlock() - db.mmaplock.RLock() - defer db.mmaplock.RUnlock() + db.mmaplock.Lock() + defer db.mmaplock.Unlock() return db.close() } @@ -1007,6 +1029,13 @@ type Options struct { // under normal operation, but requires a full database re-sync during recovery. NoFreelistSync bool + // FreelistType sets the backend freelist type. There are two options. Array which is simple but endures + // dramatic performance degradation if database is large and framentation in freelist is common. + // The alternative one is using hashmap, it is faster in almost all circumstances + // but it doesn't guarantee that it offers the smallest page id available. In normal case it is safe. + // The default type is array + FreelistType FreelistType + // Open database in read-only mode. Uses flock(..., LOCK_SH |LOCK_NB) to // grab a shared lock (UNIX). ReadOnly bool @@ -1031,13 +1060,18 @@ type Options struct { // set directly on the DB itself when returned from Open(), but this option // is useful in APIs which expose Options but not the underlying DB. NoSync bool + + // OpenFile is used to open files. It defaults to os.OpenFile. This option + // is useful for writing hermetic tests. + OpenFile func(string, int, os.FileMode) (*os.File, error) } // DefaultOptions represent the options used if nil options are passed into Open(). // No timeout is used which will cause Bolt to wait indefinitely for a lock. var DefaultOptions = &Options{ - Timeout: 0, - NoGrowSync: false, + Timeout: 0, + NoGrowSync: false, + FreelistType: FreelistArrayType, } // Stats represents statistics about the database. diff --git a/vendor/github.com/coreos/bbolt/doc.go b/vendor/github.com/coreos/bbolt/doc.go index cc937845d..95f25f01c 100644 --- a/vendor/github.com/coreos/bbolt/doc.go +++ b/vendor/github.com/coreos/bbolt/doc.go @@ -1,5 +1,5 @@ /* -Package bolt implements a low-level key/value store in pure Go. It supports +package bbolt implements a low-level key/value store in pure Go. It supports fully serializable transactions, ACID semantics, and lock-free MVCC with multiple readers and a single writer. Bolt can be used for projects that want a simple data store without the need to add large dependencies such as @@ -41,4 +41,4 @@ point to different data or can point to invalid memory which will cause a panic. */ -package bolt +package bbolt diff --git a/vendor/github.com/coreos/bbolt/errors.go b/vendor/github.com/coreos/bbolt/errors.go index a3620a3eb..48758ca57 100644 --- a/vendor/github.com/coreos/bbolt/errors.go +++ b/vendor/github.com/coreos/bbolt/errors.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import "errors" diff --git a/vendor/github.com/coreos/bbolt/freelist.go b/vendor/github.com/coreos/bbolt/freelist.go index 266f15429..587b8cc02 100644 --- a/vendor/github.com/coreos/bbolt/freelist.go +++ b/vendor/github.com/coreos/bbolt/freelist.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "fmt" @@ -14,22 +14,54 @@ type txPending struct { lastReleaseBegin txid // beginning txid of last matching releaseRange } +// pidSet holds the set of starting pgids which have the same span size +type pidSet map[pgid]struct{} + // freelist represents a list of all pages that are available for allocation. // It also tracks pages that have been freed but are still in use by open transactions. type freelist struct { - ids []pgid // all free and available free page ids. - allocs map[pgid]txid // mapping of txid that allocated a pgid. - pending map[txid]*txPending // mapping of soon-to-be free page ids by tx. - cache map[pgid]bool // fast lookup of all free and pending page ids. + freelistType FreelistType // freelist type + ids []pgid // all free and available free page ids. + allocs map[pgid]txid // mapping of txid that allocated a pgid. + pending map[txid]*txPending // mapping of soon-to-be free page ids by tx. + cache map[pgid]bool // fast lookup of all free and pending page ids. + freemaps map[uint64]pidSet // key is the size of continuous pages(span), value is a set which contains the starting pgids of same size + forwardMap map[pgid]uint64 // key is start pgid, value is its span size + backwardMap map[pgid]uint64 // key is end pgid, value is its span size + allocate func(txid txid, n int) pgid // the freelist allocate func + free_count func() int // the function which gives you free page number + mergeSpans func(ids pgids) // the mergeSpan func + getFreePageIDs func() []pgid // get free pgids func + readIDs func(pgids []pgid) // readIDs func reads list of pages and init the freelist } // newFreelist returns an empty, initialized freelist. -func newFreelist() *freelist { - return &freelist{ - allocs: make(map[pgid]txid), - pending: make(map[txid]*txPending), - cache: make(map[pgid]bool), +func newFreelist(freelistType FreelistType) *freelist { + f := &freelist{ + freelistType: freelistType, + allocs: make(map[pgid]txid), + pending: make(map[txid]*txPending), + cache: make(map[pgid]bool), + freemaps: make(map[uint64]pidSet), + forwardMap: make(map[pgid]uint64), + backwardMap: make(map[pgid]uint64), } + + if freelistType == FreelistMapType { + f.allocate = f.hashmapAllocate + f.free_count = f.hashmapFreeCount + f.mergeSpans = f.hashmapMergeSpans + f.getFreePageIDs = f.hashmapGetFreePageIDs + f.readIDs = f.hashmapReadIDs + } else { + f.allocate = f.arrayAllocate + f.free_count = f.arrayFreeCount + f.mergeSpans = f.arrayMergeSpans + f.getFreePageIDs = f.arrayGetFreePageIDs + f.readIDs = f.arrayReadIDs + } + + return f } // size returns the size of the page after serialization. @@ -47,8 +79,8 @@ func (f *freelist) count() int { return f.free_count() + f.pending_count() } -// free_count returns count of free pages -func (f *freelist) free_count() int { +// arrayFreeCount returns count of free pages(array version) +func (f *freelist) arrayFreeCount() int { return len(f.ids) } @@ -69,12 +101,12 @@ func (f *freelist) copyall(dst []pgid) { m = append(m, txp.ids...) } sort.Sort(m) - mergepgids(dst, f.ids, m) + mergepgids(dst, f.getFreePageIDs(), m) } -// allocate returns the starting page id of a contiguous list of pages of a given size. +// arrayAllocate returns the starting page id of a contiguous list of pages of a given size. // If a contiguous block cannot be found then 0 is returned. -func (f *freelist) allocate(txid txid, n int) pgid { +func (f *freelist) arrayAllocate(txid txid, n int) pgid { if len(f.ids) == 0 { return 0 } @@ -160,8 +192,7 @@ func (f *freelist) release(txid txid) { delete(f.pending, tid) } } - sort.Sort(m) - f.ids = pgids(f.ids).merge(m) + f.mergeSpans(m) } // releaseRange moves pending pages allocated within an extent [begin,end] to the free list. @@ -194,8 +225,7 @@ func (f *freelist) releaseRange(begin, end txid) { delete(f.pending, tid) } } - sort.Sort(m) - f.ids = pgids(f.ids).merge(m) + f.mergeSpans(m) } // rollback removes the pages from a given pending tx. @@ -222,8 +252,7 @@ func (f *freelist) rollback(txid txid) { } // Remove pages from pending list and mark as free if allocated by txid. delete(f.pending, txid) - sort.Sort(m) - f.ids = pgids(f.ids).merge(m) + f.mergeSpans(m) } // freed returns whether a given page is in the free list. @@ -249,23 +278,27 @@ func (f *freelist) read(p *page) { f.ids = nil } else { ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx : idx+count] - f.ids = make([]pgid, len(ids)) - copy(f.ids, ids) + // copy the ids, so we don't modify on the freelist page directly + idsCopy := make([]pgid, count) + copy(idsCopy, ids) // Make sure they're sorted. - sort.Sort(pgids(f.ids)) - } + sort.Sort(pgids(idsCopy)) - // Rebuild the page cache. - f.reindex() + f.readIDs(idsCopy) + } } -// read initializes the freelist from a given list of ids. -func (f *freelist) readIDs(ids []pgid) { +// arrayReadIDs initializes the freelist from a given list of ids. +func (f *freelist) arrayReadIDs(ids []pgid) { f.ids = ids f.reindex() } +func (f *freelist) arrayGetFreePageIDs() []pgid { + return f.ids +} + // write writes the page ids onto a freelist page. All free and pending ids are // saved to disk since in the event of a program crash, all pending ids will // become free. @@ -307,22 +340,42 @@ func (f *freelist) reload(p *page) { // Check each page in the freelist and build a new available freelist // with any pages not in the pending lists. var a []pgid - for _, id := range f.ids { + for _, id := range f.getFreePageIDs() { if !pcache[id] { a = append(a, id) } } - f.ids = a - // Once the available list is rebuilt then rebuild the free cache so that - // it includes the available and pending free pages. - f.reindex() + f.readIDs(a) +} + +// noSyncReload reads the freelist from pgids and filters out pending items. +func (f *freelist) noSyncReload(pgids []pgid) { + // Build a cache of only pending pages. + pcache := make(map[pgid]bool) + for _, txp := range f.pending { + for _, pendingID := range txp.ids { + pcache[pendingID] = true + } + } + + // Check each page in the freelist and build a new available freelist + // with any pages not in the pending lists. + var a []pgid + for _, id := range pgids { + if !pcache[id] { + a = append(a, id) + } + } + + f.readIDs(a) } // reindex rebuilds the free cache based on available and pending free lists. func (f *freelist) reindex() { - f.cache = make(map[pgid]bool, len(f.ids)) - for _, id := range f.ids { + ids := f.getFreePageIDs() + f.cache = make(map[pgid]bool, len(ids)) + for _, id := range ids { f.cache[id] = true } for _, txp := range f.pending { @@ -331,3 +384,9 @@ func (f *freelist) reindex() { } } } + +// arrayMergeSpans try to merge list of pages(represented by pgids) with existing spans but using array +func (f *freelist) arrayMergeSpans(ids pgids) { + sort.Sort(ids) + f.ids = pgids(f.ids).merge(ids) +} diff --git a/vendor/github.com/coreos/bbolt/freelist_hmap.go b/vendor/github.com/coreos/bbolt/freelist_hmap.go new file mode 100644 index 000000000..02ef2be04 --- /dev/null +++ b/vendor/github.com/coreos/bbolt/freelist_hmap.go @@ -0,0 +1,178 @@ +package bbolt + +import "sort" + +// hashmapFreeCount returns count of free pages(hashmap version) +func (f *freelist) hashmapFreeCount() int { + // use the forwardmap to get the total count + count := 0 + for _, size := range f.forwardMap { + count += int(size) + } + return count +} + +// hashmapAllocate serves the same purpose as arrayAllocate, but use hashmap as backend +func (f *freelist) hashmapAllocate(txid txid, n int) pgid { + if n == 0 { + return 0 + } + + // if we have a exact size match just return short path + if bm, ok := f.freemaps[uint64(n)]; ok { + for pid := range bm { + // remove the span + f.delSpan(pid, uint64(n)) + + f.allocs[pid] = txid + + for i := pgid(0); i < pgid(n); i++ { + delete(f.cache, pid+i) + } + return pid + } + } + + // lookup the map to find larger span + for size, bm := range f.freemaps { + if size < uint64(n) { + continue + } + + for pid := range bm { + // remove the initial + f.delSpan(pid, uint64(size)) + + f.allocs[pid] = txid + + remain := size - uint64(n) + + // add remain span + f.addSpan(pid+pgid(n), remain) + + for i := pgid(0); i < pgid(n); i++ { + delete(f.cache, pid+pgid(i)) + } + return pid + } + } + + return 0 +} + +// hashmapReadIDs reads pgids as input an initial the freelist(hashmap version) +func (f *freelist) hashmapReadIDs(pgids []pgid) { + f.init(pgids) + + // Rebuild the page cache. + f.reindex() +} + +// hashmapGetFreePageIDs returns the sorted free page ids +func (f *freelist) hashmapGetFreePageIDs() []pgid { + count := f.free_count() + if count == 0 { + return nil + } + + m := make([]pgid, 0, count) + for start, size := range f.forwardMap { + for i := 0; i < int(size); i++ { + m = append(m, start+pgid(i)) + } + } + sort.Sort(pgids(m)) + + return m +} + +// hashmapMergeSpans try to merge list of pages(represented by pgids) with existing spans +func (f *freelist) hashmapMergeSpans(ids pgids) { + for _, id := range ids { + // try to see if we can merge and update + f.mergeWithExistingSpan(id) + } +} + +// mergeWithExistingSpan merges pid to the existing free spans, try to merge it backward and forward +func (f *freelist) mergeWithExistingSpan(pid pgid) { + prev := pid - 1 + next := pid + 1 + + preSize, mergeWithPrev := f.backwardMap[prev] + nextSize, mergeWithNext := f.forwardMap[next] + newStart := pid + newSize := uint64(1) + + if mergeWithPrev { + //merge with previous span + start := prev + 1 - pgid(preSize) + f.delSpan(start, preSize) + + newStart -= pgid(preSize) + newSize += preSize + } + + if mergeWithNext { + // merge with next span + f.delSpan(next, nextSize) + newSize += nextSize + } + + f.addSpan(newStart, newSize) +} + +func (f *freelist) addSpan(start pgid, size uint64) { + f.backwardMap[start-1+pgid(size)] = size + f.forwardMap[start] = size + if _, ok := f.freemaps[size]; !ok { + f.freemaps[size] = make(map[pgid]struct{}) + } + + f.freemaps[size][start] = struct{}{} +} + +func (f *freelist) delSpan(start pgid, size uint64) { + delete(f.forwardMap, start) + delete(f.backwardMap, start+pgid(size-1)) + delete(f.freemaps[size], start) + if len(f.freemaps[size]) == 0 { + delete(f.freemaps, size) + } +} + +// initial from pgids using when use hashmap version +// pgids must be sorted +func (f *freelist) init(pgids []pgid) { + if len(pgids) == 0 { + return + } + + size := uint64(1) + start := pgids[0] + + if !sort.SliceIsSorted([]pgid(pgids), func(i, j int) bool { return pgids[i] < pgids[j] }) { + panic("pgids not sorted") + } + + f.freemaps = make(map[uint64]pidSet) + f.forwardMap = make(map[pgid]uint64) + f.backwardMap = make(map[pgid]uint64) + + for i := 1; i < len(pgids); i++ { + // continuous page + if pgids[i] == pgids[i-1]+1 { + size++ + } else { + f.addSpan(start, size) + + size = 1 + start = pgids[i] + } + } + + // init the tail + if size != 0 && start != 0 { + f.addSpan(start, size) + } +} diff --git a/vendor/github.com/coreos/bbolt/go.mod b/vendor/github.com/coreos/bbolt/go.mod new file mode 100644 index 000000000..8386eeaea --- /dev/null +++ b/vendor/github.com/coreos/bbolt/go.mod @@ -0,0 +1,3 @@ +module go.etcd.io/bbolt + +go 1.12 diff --git a/vendor/github.com/coreos/bbolt/node.go b/vendor/github.com/coreos/bbolt/node.go index f4ce240ed..6c3fa553e 100644 --- a/vendor/github.com/coreos/bbolt/node.go +++ b/vendor/github.com/coreos/bbolt/node.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "bytes" diff --git a/vendor/github.com/coreos/bbolt/page.go b/vendor/github.com/coreos/bbolt/page.go index cde403ae8..bca9615f0 100644 --- a/vendor/github.com/coreos/bbolt/page.go +++ b/vendor/github.com/coreos/bbolt/page.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "fmt" diff --git a/vendor/github.com/coreos/bbolt/tx.go b/vendor/github.com/coreos/bbolt/tx.go index 41a9bc619..2df7688c2 100644 --- a/vendor/github.com/coreos/bbolt/tx.go +++ b/vendor/github.com/coreos/bbolt/tx.go @@ -1,4 +1,4 @@ -package bolt +package bbolt import ( "fmt" @@ -254,17 +254,36 @@ func (tx *Tx) Rollback() error { if tx.db == nil { return ErrTxClosed } - tx.rollback() + tx.nonPhysicalRollback() return nil } +// nonPhysicalRollback is called when user calls Rollback directly, in this case we do not need to reload the free pages from disk. +func (tx *Tx) nonPhysicalRollback() { + if tx.db == nil { + return + } + if tx.writable { + tx.db.freelist.rollback(tx.meta.txid) + } + tx.close() +} + +// rollback needs to reload the free pages from disk in case some system error happens like fsync error. func (tx *Tx) rollback() { if tx.db == nil { return } if tx.writable { tx.db.freelist.rollback(tx.meta.txid) - tx.db.freelist.reload(tx.db.page(tx.db.meta().freelist)) + if !tx.db.hasSyncedFreelist() { + // Reconstruct free page list by scanning the DB to get the whole free page list. + // Note: scaning the whole db is heavy if your db size is large in NoSyncFreeList mode. + tx.db.freelist.noSyncReload(tx.db.freepages()) + } else { + // Read free page list from freelist page. + tx.db.freelist.reload(tx.db.page(tx.db.meta().freelist)) + } } tx.close() } @@ -315,7 +334,7 @@ func (tx *Tx) Copy(w io.Writer) error { // If err == nil then exactly tx.Size() bytes will be written into the writer. func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) { // Attempt to open reader with WriteFlag - f, err := os.OpenFile(tx.db.path, os.O_RDONLY|tx.WriteFlag, 0) + f, err := tx.db.openFile(tx.db.path, os.O_RDONLY|tx.WriteFlag, 0) if err != nil { return 0, err } @@ -369,7 +388,7 @@ func (tx *Tx) WriteTo(w io.Writer) (n int64, err error) { // A reader transaction is maintained during the copy so it is safe to continue // using the database while a copy is in progress. func (tx *Tx) CopyFile(path string, mode os.FileMode) error { - f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) + f, err := tx.db.openFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) if err != nil { return err } -- 2.25.0