ChunkServer.prp

#
# $Id$
#
# Copyright 2008-2012,2016 Quantcast Corporation. All rights reserved.
#
# Author: Mike Ovsiannikov
#
# This file is part of Kosmos File System (KFS).
#
# Licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License.
#
# The chunk server configuration.
# Additional information can be found in
# https://github.com/quantcast/qfs/wiki/Configuration-Reference.

# The following parameters set must be specified at startup time. Other
# parameters can be changed at runtime, and it usually more convenient to
# specify these in meta server configuration. The meta server broadcasts the
# corresponding chunk server parameters to all connected chunk chunk servers.

# The meta server location.
chunkServer.metaServer.hostname = localhost
chunkServer.metaServer.port     = 30000

# Client connection listener ip address to bind to.
# Use :: to bind to ipv6 address any.
# Default is empty, treaated as 0.0.0.0 ipv4 address any, unless the following
# parameter chunkServer.clientIpV6Only set to 1
# chunkServer.clientIp =

# Accept ipv4 chunk servers connections.
# Default is 0, enable acception ipv4 connection. Only has effect if
# chunkServer.clientIp left empty, or set to ::
# chunkServer.clientIpV6Only = 0

# Port to open for client connections
chunkServer.clientPort = 22000

# IPv[46] chink server addresses used by clients to connect to the chunk server.
# The IP does not have to be assigned to any local NICs. The IP is valid as long
# as the clients can reach chunk server process using this IP.
#
# In the case when the chunk server is on the same host as the meta server, but
# the clients aren't, the server IP must be specified. Setting
# cnchunkServer.metaServer.hostname to the client "visible" IP might also work.
#
# This also allows to work with NAT between the clients, and chunk and meta
# servers.
#
# The chunk server IP address by default is assigned by chunk server, and
# updated every time chunk server connects to the meta server. The IP address is
# obtained by calling getsockname() on the meta server connection socket, i.e.
# local IP address as perceived by the chunk server.
#
# Setting this parameter to 0.0.0.0 changes the default IP address assignment
# behavior. In this mode, meta server obtains chunk server IP by invoking
# getpeername() with the chunk server connection socket.
# This mode is intended to handle the cases with NAT between chunk, meta
# server, and clients, when the "local" IP address obtained by the default
# method cannot be used by clients to connect to the chunk servers (for example
# Kubernetes).
#
# Default is empty string.
# chunkServer.hostname =

# Chunk server's rack id. Has effect only if meta server rack prefixes
# (metaServer.rackPrefixes) are not set, or chunk server's ip do not match
# any of the prefix.
# Valid rack id range is from 0 to 65535. Rack ids outside of this range are
# treated by the meta server as undefined / unassigned, i.e. the same as -1.
# Default is no rack assigned.
# chunkServer.rackId = -1

# Chunk server's node id. If set to non empty string, used as network node /
# host identifier to determine if chunk server and client are co-located on the
# same network node. Takes precedence over client and chunk server IP addresses.
# Intended to be used in the cases where NAT and / or docker containers makes IP
# addresses comparison meaningless for this purpose.
#
# The values with FILE: prefix handled by setting node id to the md5 of the
# file content.
# For example the following would set node id to
# d41d8cd98f00b204e9800998ecf8427e
# chunkServer.nodeId = FILE:/dev/null
#
# The client.nodeId parameter must also be set by the QFS client configuration
# to the same value as chunk server nodeId in order to indicate that both chunk
# server and client are co-located on the same node.
#
# Default is empty string.
# chunkServer.nodeId =

# Space separated list of directories to store chunks (blocks).
# Usually one directory per physical disk. More than one directory can
# be used in the cases where the host file system has problems / limitations
# large directories.
# The directories that are "not available" (don't exists, io errors,
# "evacuate.done" file exists, etc.) at a given moment directories are
# periodically scanned.
# If the directory becomes available while chunk server is running, chunk server
# deletes all chunk files in this directory (if any), and starts using this
# directory.
# All available directories are periodically scanned, if the directory becomes
# "unavailable" all chunks in this directory are declared lost, and the
# gets added to "not available" directories which are periodically scanned as
# described the above.
chunkServer.chunkDir = chunks

# Number of io threads (max. number of disk io requests in flight) per host file
# system.
# The typical setup is to have one host file system per physical disk.
# Even if raid controller is available jbod configuration is recommended,
# leaving the failure handling and striping to the distributed file system.
# The default is 2.
# With large requests (~1MB) two io requests in flight should be sufficient.
# chunkServer.diskQueue.threadCount = 2

# Number of "client" / network io threads used to service "client" requests,
# including requests from other chunk servers, handle synchronous replication,
# chunk re-replication, and chunk RS recovery. Client threads allow to use more
# than one cpu to perform network io, encryption / decryption, request parsing,
# checksum and RS recovery calculation.
# Multiple "client" threads might help to increase network throughput. With
# modern Intel CPUs, maximum single cpu throughput can be expected roughly
# 300MB/sec (under 3Gbps network) with 1MB average write request size, and
# approximately 150MB/sec with network encryption enabled.
# This parameter has effect only on startup. Please see meta server
# configuration for related parameters description. The parameters described in
# the meta server configuration can be changed at any time, and the changes
# will have effect without chunk server restart.
# Default is 0. All network io, replication and recovery is handled by the
# "main" thread.
# chunkServer.clientThreadCount = 0

# Set client thread affinity to CPU, starting from the specified CPU index. The
# first cpu index is 0.
# If the number of CPUs is less than start index plus the number of threads, the
# the threads affinity at start index plus CPU count will be set to the last
# CPU.
# Setting affinity might help to reduce processor ram cache misses.
# The parameter has effect only on startup, and has effect only on Linux OS.
# Default is -1, no cpu affinity set.
# chunkServer.clientThreadFirstCpuIndex = -1

# Set the cluster / fs key, to protect against data loss and "data corruption"
# due to connecting to a meta server hosting different file system.
chunkServer.clusterKey = my-fs-unique-identifier

# Redirect stderror and out into /dev/null to handle the case where one or both
# are written into a file and the host file system / disk where the file resides
# exhibiting noticeable io stalls, or completely unavailable..
# Normally all the log message output performed by the message writer (thread)
# that deals with log io stall by dropping log messages. This redirection is for
# extra safety to handle the case if some library function attempting to write
# into stdout / stderror.
chunkServer.stdout = /dev/null
chunkServer.stderr = /dev/null

# The following parameter defines max size of io buffer memory used by the chunk
# server.
# The value set here, 128K means 128K * 4K buffer = 512M of buffers
# The default values are 64K (128MB) for 32 bit build, and 192K (768MB) for 64
# bit build.
# The optimal amount of memory depends on the number of disks in the system, and
# the io (read, write) concurrency -- the number of concurrent "clients". The
# memory should be increased if large number of concurrent write appenders is
# expected. Ideally the disk io request should be around 1MB, thus for each
# chunk opened for append at least 1MB of io buffers is recommended.
chunkServer.ioBufferPool.partitionBufferCount = 131072

# Buffer manager portion of all io buffers.
# This value defines max amount of io buffers that can be used for servicing
# "client" requests, chunk re-replication, and recovery.
# The remaining (1 - chunkServer.bufferManager.maxRatio) used for write append
# buffering and other "internal" purposes.
# Default is 0.4 or 40%
# chunkServer.bufferManager.maxRatio = 0.4

# Set the following to 1 if no backward compatibility with the previous kfs
# releases required. 0 is the default.
# When set to 0 the 0 header checksum (all 8 bytes must be 0) is treated as
# no checksum and therefore no chunk file header checksum verification
# performed.
# The downside of the compatibility mode is that chunk server might not detect
# the cases where the host os zero fills the data during the host file system
# recovery / journal / transaction log replay,
# thus the data loss / corruption problem might not be detected.
# chunkServer.requireChunkHeaderChecksum = 0

# If set to a value greater than 0 then locked memory limit will be set to the
# specified value, and mlock(MCL_CURRENT|MCL_FUTURE) invoked.
# On linux running under non root user setting locked memory "hard" limit
# greater or equal to the specified value required. ulimit -l can be used for
# example.
# Default is 0 -- no memory locking.
# chunkServer.maxLockedMemory = 0

# Mlock io buffers memory at startup, if set to non 0.
# Default is 0 -- no io buffer memory locking.
# chunkServer.ioBufferPool.lockMemory = 0

# ---------------------------------- Message log. ------------------------------

# Set reasonable log level, and other message log parameter to handle the case
# when meta server not available, or doesn't accept this chunk server for any
# reason.
# The chunk servers message log configuration parameters including log level
# level can be changed in the meta server configuration file.
chunkServer.msgLogWriter.logLevel = INFO

# Colon (:) separated file name prefixes to store log segments.
# Default is empty list. The default is to use file name from the command line
# or if none specified write into file descriptor 2 -- stderror.
# chunkServer.msgLogWriter.logFilePrefixes =

# Maximum log segment size.
# Default is -1 -- unlimited.
# chunkServer.msgLogWriter.maxLogFileSize = -1

# Maximum number of log segments.
# Default is -1 -- unlimited.
# chunkServer.msgLogWriter.maxLogFiles = -1

# Max. time to wait for the log buffer to become available.
# When wait is enabled the request processing thread will wait for the log
# buffer disk io to complete. If the disk subsystem cannot handle the
# amount of logging it will slow down the request processing.
# For chunk servers keeping the default is strongly recommended to minimize
# dependency on the host's disk subsystem reliability and performance.
# Default is -1. Do not wait, drop log record instead.
# chunkServer.msgLogWriter.waitMicroSec = -1

# Minimal interval in seconds to emit chunk server counters into chunk server
# message log.
# The counters are emitted in the following form format
# ===counters: ;key-name1=key-value1,key-name2=key-value2,...
# Setting this parameter to a negative value turns of this functionality.
# Setting to 0 turns off counters logging when chunk server is not connected to
# the meta server, in this case the counters are logged on every meta server
# heartbeat request with counters flag on.
# Default is 60 seconds.
# chunkServer.logChunkServerCountersInterval = 60

# Counters log level. The message log level must set to this level or greater.
# Default is NOTICE
# chunkServer.logChunkServerCountersLogLevel = NOTICE

# -------------------- Chunk and meta server authentication. -------------------
# By default chunk and meta server authentication is off.
#
# If any of the following meta authentication methods is configured then chunk
# server requires QFS client connection to be authenticated. In other words, the
# QFS client, and other chunk server acting as a client, must obtain from the
# meta server chunk server access token and corresponding key and use this token
# and the key to authenticate with the chunk server.

# ================= X509 authentication ========================================
#
# Chunk server's X509 certificate file in PEM format.
# chunkserver.meta.auth.X509.X509PemFile =

# Password if X509 PEM file is encrypted.
# chunkserver.meta.auth.X509.X509Password =

# Chunk server's private key file.
# chunkserver.meta.auth.X509.PKeyPemFile =

# Password if private key PEM file is encrypted.
# chunkserver.meta.auth.X509.PKeyPassword =

# Certificate authorities file. Used for both meta server certificate
# validation and to create certificate chain with chunk server's X509
# certificate.
# chunkserver.meta.auth.X509.CAFile =

# Certificate authorities directory can be used in addition to CAFile.
# For more detailed information please see SSL_CTX_load_verify_locations manual
# page. CAFile/CADir corresponds to CAfile/CApath in the man page.
# chunkserver.meta.auth.X509.CADir =

# If set (the default) verify peer certificate, and declare error if peer, i.e.
# meta server, does not preset "trusted" valid X509 certificate.
# Default is on.
# chunkserver.meta.auth.X509.verifyPeer = 1

# OpenSSL cipher configuration.
# chunkserver.meta.auth.X509.cipher = !ADH:!AECDH:!MD5:HIGH:@STRENGTH

# The long integer value passed to SSL_CTX_set_options() call.
# See open ssl documentation for details.
# Default is the integer value that corresponds to SSL_OP_NO_COMPRESSION
# chunkserver.meta.auth.X509.options =

# ================= Kerberos authentication ====================================
#
# Kerberos principal: service/host@realm

# Meta server's Kerberos principal [service/host@realm] service name part.
# chunkserver.meta.auth.krb5.service =

# Meta server's Kerberos principal [service/host@realm] host name part.
# chunkserver.meta.auth.krb5.host =

# Kerberos keytab file with the key(s) that corresponds to the chunk server's
# principal.
# chunkserver.meta.auth.krb5.keytab =

# Chunk server's kerberos principal. krb5_parse_name() used to convert the name
# into the Kerberos 5 internal principal representation.
# chunkserver.meta.auth.krb5.clientName =

# Force Kerberos client cache initialization during intialization.
# Default is off.
# chunkserver.meta.auth.krb5.initClientCache = 0

# OpenSSL cipher configuration for TLS-PSK authentication method. This method
# is used with delegation and with Kerberos authentication.
# chunkserver.meta.auth.psk.cipherpsk = !ADH:!AECDH:!MD5:!3DES:PSK:@STRENGTH

# The long integer value passed to SSL_CTX_set_options() call.
# See open ssl documentation for details.
# Default is the integer value that corresponds to the logical OR of
# SSL_OP_NO_COMPRESSION and SSL_OP_NO_TICKET
# metaServer.clientAuthentication.psk.options =

# ================= PSK authentication =========================================
#
# PSK chunk server authentication is intended only for testing and possibly for
# small [test] clusters with very few chunk servers, where the same
# authentication credentials [PSK "key"] are used for for all chunk servers.

# Chunk server's PSK key, Must be identical to the key the meta server
# configured with. See metaServer.CSAuthentication.psk.key parameter description
# in the annotated meta server configuration file.
# chunkserver.meta.auth.psk.key =

# Chunk server's PSK key id. See metaServer.CSAuthentication.psk.keyId parameters
# description in the annotated meta server configuration file.
# chunkserver.meta.auth.psk.keyId =

#-------------------------------------------------------------------------------

# ==================== AWS S3 object store =====================================
#
# The following parameters take effect only on chunk server startup.
# For parameters that can be changed at run time, please see corresponding
# section in meta server annotated configuration file.
#
# Object store directory. The part (suffix) after s3:// is treated as
# configuration suffix appended between chunkServer.diskQueue. prefix and S3
# specific configuration names. For example in s3://bucket_name. "bucket_name."
# is configuration suffix. If more than one directory specified (like chunk
# directories), the directories must be assigned to different storage tiers. The
# chunk server allows only one storage directory per tier, as otherwise it has
# to look into multiple directories to find object block (chunk). With more than
# single directory per tier chunk server will return error for object store
# reads and writes until the configuration is fixed by assigning directores to
# different tiers. Tiers assignment can be updated by the meta server. The
# remaining S3 configuration can be set by the meta server. The meta server
# annotated configuration file has corresponding section with
# chunkServer.diskQueue.<suffix>. description.
# Default is no object directories.
# chunkServer.objectDir =

# The following is an example how to configure chunk server with most memory
# assigned for S3 writes. This example uses parameters described the prior
# section of this file..
#
# Give chunk server 3GB of buffer space (4096 buffers):
# chunkServer.ioBufferPool.partitionBufferCount = 786432
#
# Use 256MB as request processing buffers:
# chunkServer.bufferManager.maxRatio            = 0.08334
#
# Give something to write append.
# chunkServer.recAppender.bufferLimitRatio      = 0.00001
#
# Use most of the 3GB as S3 write buffers:
# With this chunk server can have max 269 object store blocks simultaneously
# opened for write. This assumes the default 5MB minimum S3 part of multipart
# upload size. Two 5MB buffers plus one 64KB checksum block must be kept in
# memory for each writable object store blocks.
# 786432*4096*(1-0.08334)*0.96/(2^20*10+64*2^10) => 268
# Leaving the remaining 112MB "slack" for request handling etc.
# 786432*4096*(1-0.08334)*(1-0.96-0.00001)/2^20 => 112
# chunkServer.objStoreBufferDataRatio           = 0.96

# List of replicated meta server chunk server listeners network locations.
# Each location is a pair of IP address and port.
# The parameter is intended to be used in cases where DNS record with the list
# of meta server nodes IP addresses cannot be created.
# chunkServer.meta.nodes =