config.cc

/*
 * Copyright (C) 2015 ScyllaDB
 *
 */

/*
 * This file is part of Scylla.
 *
 * Scylla is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Scylla is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Scylla.  If not, see <http://www.gnu.org/licenses/>.
 */

#include <yaml-cpp/yaml.h>
#include <boost/program_options.hpp>
#include <unordered_map>
#include <regex>
#include "config.hh"
#include "core/file.hh"
#include "core/reactor.hh"
#include "core/shared_ptr.hh"
#include "core/fstream.hh"
#include "core/do_with.hh"
#include "core/print.hh"
#include "utils/log.hh"
#include <boost/any.hpp>

static logging::logger clogger("config");

redis::config::config()
:
        // Initialize members to defaults.
#define _mk_init(name, type, deflt, status, desc, ...)      \
        name(deflt),
    _make_config_values(_mk_init)
    _dummy(0)
{}

namespace bpo = boost::program_options;

namespace redis {
// Special "validator" for boost::program_options to allow reading options
// into an unordered_map<string, string> (we have in config.hh a bunch of
// those). This validator allows the parameter of each option to look like
// 'key=value'. It also allows multiple occurrences of this option to add
// multiple entries into the map. "String" can be any time which can be
// converted from std::string, e.g., sstring.
static void validate(boost::any& out, const std::vector<std::string>& in,
        redis::string_map*, int) {
    if (out.empty()) {
        out = boost::any(redis::string_map());
    }
    auto* p = boost::any_cast<redis::string_map>(&out);
    for (const auto& s : in) {
        auto i = s.find_first_of('=');
        if (i == std::string::npos) {
            throw boost::program_options::invalid_option_value(s);
        }
        (*p)[sstring(s.substr(0, i))] = sstring(s.substr(i+1));
    }
}
}

namespace YAML {
/*
 * Add converters as needed here...
 */
template<>
struct convert<sstring> {
    static Node encode(sstring rhs) {
        auto p = rhs.c_str();
        return convert<const char *>::encode(p);
    }
    static bool decode(const Node& node, sstring& rhs) {
        std::string tmp;
        if (!convert<std::string>::decode(node, tmp)) {
            return false;
        }
        rhs = tmp;
        return true;
    }
};

template <>
struct convert<redis::config::string_list> {
    static Node encode(const redis::config::string_list& rhs) {
        Node node(NodeType::Sequence);
        for (auto& s : rhs) {
            node.push_back(convert<sstring>::encode(s));
        }
        return node;
    }
    static bool decode(const Node& node, redis::config::string_list& rhs) {
        if (!node.IsSequence()) {
            return false;
        }
        rhs.clear();
        for (auto& n : node) {
            sstring tmp;
            if (!convert<sstring>::decode(n,tmp)) {
                return false;
            }
            rhs.push_back(tmp);
        }
        return true;
    }
};

template<>
struct convert<redis::string_map> {
    static Node encode(const redis::string_map& rhs) {
        Node node(NodeType::Map);
        for (auto& p : rhs) {
            node.force_insert(p.first, p.second);
        }
        return node;
    }
    static bool decode(const Node& node, redis::string_map& rhs) {
        if (!node.IsMap()) {
            return false;
        }
        rhs.clear();
        for (auto& n : node) {
            rhs[n.first.as<sstring>()] = n.second.as<sstring>();
        }
        return true;
    }
};

template<>
struct convert<redis::config::seed_provider_type> {
    static Node encode(const redis::config::seed_provider_type& rhs) {
        throw std::runtime_error("should not reach");
    }
    static bool decode(const Node& node, redis::config::seed_provider_type& rhs) {
        if (!node.IsSequence()) {
            return false;
        }
        rhs = redis::config::seed_provider_type();
        for (auto& n : node) {
            if (!n.IsMap()) {
                continue;
            }
            for (auto& n2 : n) {
                if (n2.first.as<sstring>() == "class_name") {
                    rhs.class_name = n2.second.as<sstring>();
                }
                if (n2.first.as<sstring>() == "parameters") {
                    auto v = n2.second.as<std::vector<redis::config::string_map>>();
                    if (!v.empty()) {
                        rhs.parameters = v.front();
                    }
                }
            }
        }
        return true;
    }
};


}

namespace db {
template<typename... Args>
std::basic_ostream<Args...> & operator<<(std::basic_ostream<Args...> & os, const redis::config::string_map & map) {
    int n = 0;
    for (auto& e : map) {
        if (n > 0) {
            os << ":";
        }
        os << e.first << "=" << e.second;
    }
    return os;
}

template<typename... Args>
std::basic_istream<Args...> & operator>>(std::basic_istream<Args...> & is, redis::config::string_map & map) {
    std::string str;
    is >> str;

    std::regex colon(":");

    std::sregex_token_iterator s(str.begin(), str.end(), colon, -1);
    std::sregex_token_iterator e;
    while (s != e) {
        sstring p = std::string(*s++);
        auto i = p.find('=');
        auto k = p.substr(0, i);
        auto v = i == sstring::npos ? sstring() : p.substr(i + 1, p.size());
        map.emplace(std::make_pair(k, v));
    };

    return is;
}
}

/*
 * Helper type to do compile time exclusion of Unused/invalid options from
 * command line.
 *
 * Only opts marked "used" should get a boost::opt
 *
 */
template<typename T, redis::config::value_status S>
struct do_value_opt;

template<typename T>
struct do_value_opt<T, redis::config::value_status::Used> {
    template<typename Func>
    void operator()(Func&& func, const char* name, const T& dflt, T * dst, redis::config::config_source & src, const char* desc) const {
        func(name, dflt, dst, src, desc);
    }
};

template<>
struct do_value_opt<redis::config::seed_provider_type, redis::config::value_status::Used> {
    using seed_provider_type = redis::config::seed_provider_type;
    template<typename Func>
    void operator()(Func&& func, const char* name, const seed_provider_type& dflt, seed_provider_type * dst, redis::config::config_source & src, const char* desc) const {
        func((sstring(name) + "_class_name").c_str(), dflt.class_name, &dst->class_name, src, desc);
        func((sstring(name) + "_parameters").c_str(), dflt.parameters, &dst->parameters, src, desc);
    }
};

template<typename T>
struct do_value_opt<T, redis::config::value_status::Unused> {
    template<typename... Args> void operator()(Args&&... args) const {}
};
template<typename T>
struct do_value_opt<T, redis::config::value_status::Invalid> {
    template<typename... Args> void operator()(Args&&... args) const {}
};

bpo::options_description redis::config::get_options_description() {
    bpo::options_description opts("Urchin options");
    auto init = opts.add_options();
    add_options(init);
    return std::move(opts);
}

/*
 * Our own bpo::typed_valye.
 * Only difference is that we _don't_ apply defaults (they are already applied)
 * Needed to make aliases work properly.
 */
template<class T, class charT = char>
class typed_value_ex : public bpo::typed_value<T, charT> {
public:
    typedef bpo::typed_value<T, charT> _Super;

    typed_value_ex(T* store_to)
        : _Super(store_to)
    {}
    bool apply_default(boost::any& value_store) const override {
        return false;
    }
};

template<class T>
inline typed_value_ex<T>* value_ex(T* v) {
    typed_value_ex<T>* r = new typed_value_ex<T>(v);
    return r;
}

template<class T>
inline typed_value_ex<std::vector<T>>* value_ex(std::vector<T>* v) {
    auto r = new typed_value_ex<std::vector<T>>(v);
    r->multitoken();
    return r;
}

bpo::options_description_easy_init& redis::config::add_options(bpo::options_description_easy_init& init) {
    auto opt_add =
            [&init](const char* name, const auto& dflt, auto* dst, auto& src, const char* desc) mutable {
                sstring tmp(name);
                std::replace(tmp.begin(), tmp.end(), '_', '-');
                init(tmp.c_str(),
                        value_ex(dst)->default_value(dflt)->notifier([&src](auto) mutable {
                                    src = config_source::CommandLine;
                                })
                        , desc);
            };

    // Add all used opts as command line opts
#define _add_boost_opt(name, type, deflt, status, desc, ...)      \
        do_value_opt<type, value_status::status>()(opt_add, #name, type( deflt ), &name._value, name._source, desc);
    _make_config_values(_add_boost_opt)

    auto alias_add =
            [&init](const char* name, auto& dst, const char* desc) mutable {
                init(name,
                        value_ex(&dst._value)->notifier([&dst](auto& v) mutable {
                                    dst._source = config_source::CommandLine;
                                })
                        , desc);
            };


    // Handle "old" syntax with "aliases"
    alias_add("datadir", data_file_directories, "alias for 'data-file-directories'");
    alias_add("thrift-port", rpc_port, "alias for 'rpc-port'");
    alias_add("cql-port", native_transport_port, "alias for 'native-transport-port'");

    return init;
}

// Virtual dispatch to convert yaml->data type.
struct handle_yaml {
    virtual ~handle_yaml() {};
    virtual void operator()(const YAML::Node&) = 0;
    virtual redis::config::value_status status() const = 0;
    virtual redis::config::config_source source() const = 0;
};

template<typename T, redis::config::value_status S>
struct handle_yaml_impl : public handle_yaml {
    typedef redis::config::value<T, S> value_type;

    handle_yaml_impl(value_type& v, redis::config::config_source& src) : _dst(v), _src(src) {}
    void operator()(const YAML::Node& node) override {
        _dst(node.as<T>());
        _src = redis::config::config_source::SettingsFile;
    }
    redis::config::value_status status() const override {
        return _dst.status();
    }
    redis::config::config_source source() const override {
        return _src;
    }
    value_type& _dst;
    redis::config::config_source&
        _src;
};

void redis::config::read_from_yaml(const sstring& yaml) {
    read_from_yaml(yaml.c_str());
}

void redis::config::read_from_yaml(const char* yaml) {
    std::unordered_map<sstring, std::unique_ptr<handle_yaml>> values;

#define _add_yaml_opt(name, type, deflt, status, desc, ...)      \
        values.emplace(#name, std::make_unique<handle_yaml_impl<type, value_status::status>>(name, name._source));
    _make_config_values(_add_yaml_opt)

    /*
     * Note: this is not very "half-fault" tolerant. I.e. there could be
     * yaml syntax errors that origin handles and still sets the options
     * where as we don't...
     * There are no exhaustive attempts at converting, we rely on syntax of
     * file mapping to the data type...
     */
    auto doc = YAML::Load(yaml);
    for (auto node : doc) {
        auto label = node.first.as<sstring>();
        auto i = values.find(label);
        if (i == values.end()) {
            clogger.warn("Unknown option {} ignored.", label);
            continue;
        }
        if (i->second->source() > config_source::SettingsFile) {
            clogger.debug("Option {} already set by commandline. ignored.", label);
            continue;
        }
        switch (i->second->status()) {
        case value_status::Invalid:
            clogger.warn("Option {} is not applicable. Ignoring.", label);
            continue;
        case value_status::Unused:
            clogger.warn("Option {} is not (yet) used.", label);
            break;
        default:
            break;
        }
        if (node.second.IsNull()) {
            clogger.debug("Option {}, empty value. Skipping.", label);
            continue;
        }
        // Still, a syntax error is an error warning, not a fail
        try {
            (*i->second)(node.second);
        } catch (...) {
            clogger.error("Option {}, exception while converting value.", label);
        }
    }

    for (auto& p : values) {
        if (p.second->status() > value_status::Used) {
            continue;
        }
        if (p.second->source() > config_source::None) {
            continue;
        }
        clogger.debug("Option {} not set", p.first);
    }
}

future<> redis::config::read_from_file(file f) {
    return f.size().then([this, f](size_t s) {
        return do_with(make_file_input_stream(f), [this, s](input_stream<char>& in) {
            return in.read_exactly(s).then([this](temporary_buffer<char> buf) {
               read_from_yaml(sstring(buf.begin(), buf.end()));
            });
        });
    });
}

future<> redis::config::read_from_file(const sstring& filename) {
    return open_file_dma(filename, open_flags::ro).then([this](file f) {
       return read_from_file(std::move(f));
    });
}

boost::filesystem::path redis::config::get_conf_dir() {
    using namespace boost::filesystem;

    path confdir;
    auto* cd = std::getenv("SCYLLA_CONF");
    if (cd != nullptr) {
        confdir = path(cd);
    } else {
        auto* p = std::getenv("SCYLLA_HOME");
        if (p != nullptr) {
            confdir = path(p);
        }
        confdir /= "conf";
    }

    return confdir;
}

void redis::config::check_experimental(const sstring& what) const {
    if (!experimental()) {
        throw std::runtime_error(sprint("%s is currently disabled. Start Scylla with --experimental=on to enable.", what));
    }
}