add support for ancient storage as json-rpc service

cmd/ancient-store-mem/README.md

@@ -0,0 +1,12 @@
+# Ancient Store Memory Mapped Test Util
+
+This application is intended for testing purposed only. Ancient data is stored ephemerally.
+The program expects first and only argument to be an IPC path, or, the directory
+in which a default 'mock-freezer.ipc' path should be created.
+This memory mapped ancient store can also be used as a library.
+Package 'lib' logic may be imported and used in testing contexts as well.
+
+## Usage
+```
+ancient-store-mem your-ipc-path 
+```

cmd/ancient-store-mem/lib/mem.go

@@ -0,0 +1,146 @@
+// Copyright 2020 The core-geth Authors
+// This file is part of the core-geth library.
+//
+// The core-geth library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The core-geth library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the core-geth library. If not, see <http://www.gnu.org/licenses/>.
+
+package lib
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+const (
+	freezerRemoteHashTable       = "hashes"
+	freezerRemoteHeaderTable     = "headers"
+	freezerRemoteBodiesTable     = "bodies"
+	freezerRemoteReceiptTable    = "receipts"
+	freezerRemoteDifficultyTable = "diffs"
+)
+
+var (
+	errOutOfBounds = errors.New("out of bounds")
+	errOutOfOrder  = errors.New("out of order")
+)
+
+// MemFreezerRemoteServerAPI is a mock freezer server implementation.
+type MemFreezerRemoteServerAPI struct {
+	store map[string][]byte
+	count uint64
+	mu    sync.Mutex
+}
+
+func NewMemFreezerRemoteServerAPI() *MemFreezerRemoteServerAPI {
+	return &MemFreezerRemoteServerAPI{store: make(map[string][]byte)}
+}
+
+func (r *MemFreezerRemoteServerAPI) storeKey(kind string, number uint64) string {
+	return fmt.Sprintf("%s-%d", kind, number)
+}
+
+func (f *MemFreezerRemoteServerAPI) Reset() {
+	f.count = 0
+	f.mu.Lock()
+	f.store = make(map[string][]byte)
+	f.mu.Unlock()
+}
+
+func (f *MemFreezerRemoteServerAPI) HasAncient(kind string, number uint64) (bool, error) {
+	fmt.Println("mock server called", "method=HasAncient")
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	_, ok := f.store[f.storeKey(kind, number)]
+	return ok, nil
+}
+
+func (f *MemFreezerRemoteServerAPI) Ancient(kind string, number uint64) ([]byte, error) {
+	fmt.Println("mock server called", "method=Ancient")
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	v, ok := f.store[f.storeKey(kind, number)]
+	if !ok {
+		return nil, errOutOfBounds
+	}
+	return v, nil
+}
+
+func (f *MemFreezerRemoteServerAPI) Ancients() (uint64, error) {
+	fmt.Println("mock server called", "method=Ancients")
+	return f.count, nil
+}
+
+func (f *MemFreezerRemoteServerAPI) AncientSize(kind string) (uint64, error) {
+	fmt.Println("mock server called", "method=AncientSize")
+	sum := uint64(0)
+	for k, v := range f.store {
+		if strings.HasPrefix(k, kind) {
+			sum += uint64(len(v))
+		}
+	}
+	return sum, nil
+}
+
+func (f *MemFreezerRemoteServerAPI) AppendAncient(number uint64, hash, header, body, receipt, td []byte) error {
+	fmt.Println("mock server called", "method=AppendAncient", "number=", number, "header", fmt.Sprintf("%x", header))
+	fieldNames := []string{
+		freezerRemoteHashTable,
+		freezerRemoteHeaderTable,
+		freezerRemoteBodiesTable,
+		freezerRemoteReceiptTable,
+		freezerRemoteDifficultyTable,
+	}
+	fields := [][]byte{hash, header, body, receipt, td}
+	if number != f.count {
+		return errOutOfOrder
+	}
+	f.count = number + 1
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	for i, fv := range fields {
+		kind := fieldNames[i]
+		f.store[f.storeKey(kind, number)] = fv
+	}
+	return nil
+}
+
+func (f *MemFreezerRemoteServerAPI) TruncateAncients(n uint64) error {
+	fmt.Println("mock server called", "method=TruncateAncients")
+	f.count = n
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	for k := range f.store {
+		spl := strings.Split(k, "-")
+		num, err := strconv.ParseUint(spl[1], 10, 64)
+		if err != nil {
+			return err
+		}
+		if num >= n {
+			delete(f.store, k)
+		}
+	}
+	return nil
+}
+
+func (f *MemFreezerRemoteServerAPI) Sync() error {
+	fmt.Println("mock server called", "method=Sync")
+	return nil
+}
+
+func (f *MemFreezerRemoteServerAPI) Close() error {
+	fmt.Println("mock server called", "method=Close")
+	return nil
+}

cmd/ancient-store-mem/main.go

@@ -0,0 +1,21 @@
+// Copyright 2020 The core-geth Authors
+// This file is part of the core-geth library.
+//
+// The core-geth library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The core-geth library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the core-geth library. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+func main() {
+	Execute()
+}

cmd/ancient-store-mem/root.go

@@ -0,0 +1,80 @@
+// Copyright 2020 The core-geth Authors
+// This file is part of the core-geth library.
+//
+// The core-geth library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The core-geth library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the core-geth library. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/ethereum/go-ethereum/cmd/ancient-store-mem/lib"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/spf13/cobra"
+)
+
+// rootCmd represents the base command when called without any subcommands
+var rootCmd = &cobra.Command{
+	Use:   "ancient-store-mem",
+	Short: "Memory-backed remote ancient store application",
+	Long: `Uses a memory-backed map to store ancient data.
+
+This application is intended for testing purposed only.
+Ancient data is stored ephemerally.
+
+Expects first and only argument to an IPC path, or, the directory
+in which a default 'mock-freezer.ipc' path should be created.
+
+Package 'lib' logic may be imported and used in testing contexts as well.
+`,
+
+	Run: func(cmd *cobra.Command, args []string) {
+		ipcPath := args[0]
+		fi, err := os.Stat(ipcPath)
+		if err != nil && !os.IsNotExist(err) {
+			log.Fatalln(err)
+		}
+		if fi != nil && fi.IsDir() {
+			ipcPath = filepath.Join(ipcPath, "mock-freezer.ipc")
+		}
+		listener, server, err := rpc.StartIPCEndpoint(ipcPath, nil)
+		if err != nil {
+			log.Fatalln(err)
+		}
+		defer os.Remove(ipcPath)
+		mock := lib.NewMemFreezerRemoteServerAPI()
+		err = server.RegisterName("freezer", mock)
+		if err != nil {
+			log.Fatalln(err)
+		}
+		quit := make(chan bool, 1)
+		go func() {
+			log.Println("Serving", listener.Addr())
+			log.Fatalln(server.ServeListener(listener))
+		}()
+		<-quit
+	},
+}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}

cmd/geth/chaincmd.go

@@ -80,6 +80,8 @@ The dumpgenesis command dumps the genesis block configuration in JSON format to
 		ArgsUsage: "<filename> (<filename 2> ... <filename N>) ",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.AncientFlag,
+			utils.AncientRPCFlag,
 			utils.CacheFlag,
 			utils.SyncModeFlag,
 			utils.GCModeFlag,
@@ -159,6 +161,8 @@ The export-preimages command export hash preimages to an RLP encoded stream`,
 		ArgsUsage: "<sourceChaindataDir>",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
+			utils.AncientFlag,
+			utils.AncientRPCFlag,
 			utils.CacheFlag,
 			utils.SyncModeFlag,
 			utils.FakePoWFlag,
@@ -167,8 +171,6 @@ The export-preimages command export hash preimages to an RLP encoded stream`,
 			utils.KottiFlag,
 			utils.SocialFlag,
 			utils.EthersocialFlag,
-			utils.LegacyTestnetFlag,
-			utils.RopstenFlag,
 			utils.RinkebyFlag,
 			utils.GoerliFlag,
 			utils.YoloV1Flag,

cmd/geth/main.go

@@ -68,6 +68,7 @@ var (
 		utils.LegacyBootnodesV5Flag,
 		utils.DataDirFlag,
 		utils.AncientFlag,
+		utils.AncientRPCFlag,
 		utils.KeyStoreDirFlag,
 		utils.ExternalSignerFlag,
 		utils.NoUSBFlag,

cmd/geth/usage.go

@@ -36,6 +36,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			configFileFlag,
 			utils.DataDirFlag,
 			utils.AncientFlag,
+			utils.AncientRPCFlag,
 			utils.KeyStoreDirFlag,
 			utils.NoUSBFlag,
 			utils.SmartCardDaemonPathFlag,

cmd/utils/flags.go

@@ -116,6 +116,11 @@ var (
 		Name:  "datadir.ancient",
 		Usage: "Data directory for ancient chain segments (default = inside chaindata)",
 	}
+	AncientRPCFlag = cli.StringFlag{
+		Name:  "ancient.rpc",
+		Usage: "Connect to a remote freezer via RPC. Value must an HTTP(S), WS(S), unix socket, or 'stdio' URL. Incompatible with --datadir.ancient",
+		Value: "",
+	}
 	KeyStoreDirFlag = DirectoryFlag{
 		Name:  "keystore",
 		Usage: "Directory for the keystore (default = inside the datadir)",
@@ -1556,6 +1561,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	// Ancient tx indices pruning is not available for les server now
 	// since light client relies on the server for transaction status query.
 	CheckExclusive(ctx, LegacyLightServFlag, LightServeFlag, TxLookupLimitFlag)
+
+	CheckExclusive(ctx, AncientFlag, AncientRPCFlag)
+
 	var ks *keystore.KeyStore
 	if keystores := stack.AccountManager().Backends(keystore.KeyStoreType); len(keystores) > 0 {
 		ks = keystores[0].(*keystore.KeyStore)
@@ -1578,6 +1586,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) {
 	if ctx.GlobalIsSet(AncientFlag.Name) {
 		cfg.DatabaseFreezer = ctx.GlobalString(AncientFlag.Name)
 	}
+	if ctx.GlobalIsSet(AncientRPCFlag.Name) {
+		cfg.DatabaseFreezerRemote = ctx.GlobalString(AncientRPCFlag.Name)
+	}
 
 	if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
 		Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
@@ -1858,11 +1869,14 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database {
 		err     error
 		chainDb ethdb.Database
 	)
+
+	name := "chaindata"
 	if ctx.GlobalString(SyncModeFlag.Name) == "light" {
-		name := "lightchaindata"
-		chainDb, err = stack.OpenDatabase(name, cache, handles, "")
+		name = "lightchaindata"
+	}
+	if ctx.GlobalIsSet(AncientRPCFlag.Name) {
+		chainDb, err = stack.OpenDatabaseWithFreezerRemote(name, cache, handles, ctx.GlobalString(AncientRPCFlag.Name))
 	} else {
-		name := "chaindata"
 		chainDb, err = stack.OpenDatabaseWithFreezer(name, cache, handles, ctx.GlobalString(AncientFlag.Name), "")
 	}
 	if err != nil {