diff --git a/nativelink-docs/docs/configuration/Assembling.mdx b/nativelink-docs/docs/configuration/Assembling.mdx new file mode 100644 index 000000000..ef4ca4a5d --- /dev/null +++ b/nativelink-docs/docs/configuration/Assembling.mdx @@ -0,0 +1,564 @@ +--- +sidebar_position: 2 +title: 'Assembling' +description: 'Assemble a configuration file from scratch' +--- + +# Configuration from Scratch + +NativeLink uses a JSON file as the configuration format. This section of the +documentation will breakdown the anatomy of this configuration file with step-by-step instructions on its assembly. + +Configuration JSON objects should begin with the `stores` key, followed by `workers`, `schedulers`, `servers`, and `global`. + + +```json5 +{ + "stores": {}, + "workers": [], + "schedulers": {}, + "servers": [], + "global": {} +} +``` + +This is the scaffolding for a NativeLink deployment configuration. + + +
+ Configuring Store + +### Store Name + +The value of `stores` includes top-level keys, which are user supplied names stores. The following example, defines the `AC_MAIN_STORE`. + +```json5 +{ + "stores": { + "AC_MAIN_STORE": {} + }, + "workers": [], + "schedulers": {}, + "servers": [], + "global": {}, +} +``` + +### Store Type + +Once the store has been named and its object exists, +the next key is the type of store. The options are `filesystem`, `memory`, `compression`, `dedup`, `fast_slow`, `verify`, and `experimental_s3_store`. + +```json5 +{ + "stores": { + "AC_MAIN_STORE": { + "filesystem": {} + } + }, + "workers": [], + "schedulers": {}, + "servers": [], + "global": {}, +} +``` + +### Store Options + +The contents of the object here must include `content_path`, `temp_path`, and an embedded JSON object, `eviction_policy`, which specifies the value of `max_bytes` for the store. + +```json5 +{ + "stores": { + "AC_MAIN_STORE": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + // 500mb. + "max_bytes": 500000000, + } + } + } + }, + "workers": [], + "schedulers": {}, + "servers": [], + "global": {}, +} +``` +
+ + +
+ Configuring Workers + +### Worker Array + +The value of `workers` includes a top level array that embeds the worker metadata. This array always begins with the `local` object, which is the only item permitted at this time. + +```json5 +{ + "stores": {}, + "workers": [{ + "local": {} + }], + "schedulers": {}, + "servers": [], + "global": {}, +} +``` + +### Local Worker Object Members + +The Local object has five components, `worker_api_endpoint`, `cas_fast_slow_store`, `upload_action_results`,`work_directory`, and `platform_properties`. + +```json5 +{ + "stores": {}, + "workers": [{ + "local": { + "worker_api_endpoint": { + "uri": "grpc://127.0.0.1:50061", + }, + "cas_fast_slow_store": "WORKER_FAST_SLOW_STORE", + "upload_action_result": { + "ac_store": "AC_MAIN_STORE", + }, + "work_directory": "/tmp/nativelink/work", + "platform_properties": { + "cpu_count": { + "values": ["16"], + }, + "memory_kb": { + "values": ["500000"], + }, + "network_kbps": { + "values": ["100000"], + }, + "cpu_arch": { + "values": ["x86_64"], + }, + "OSFamily": { + "values": [""] + }, + "container-image": { + "values": [""] + }, + } + } + }], + "schedulers": {}, + "servers": [], + "global": {}, +} +``` +
+ + +
+ Configuring Schedulers + + ### Scheduler Name + +The value of `schedulers` includes top-level keys, which are the user-supplied names of the schedulers. The following example, defines the `MAIN_SCHEDULER`. + +```json5 +{ + "stores": {}, + "workers": {}, + "schedulers": { + "MAIN_SCHEDULER": {} + }, + "servers": [], + "global": {}, +} +``` + +### Scheduler Type + +Once the scheduler has been named and its object exists, +the next key is the type of scheduler. The options are `simple`, `action_scheduler`, `grpc_scheduler`, `property_modifier_scheduler`, and `worker_scheduler`. + +```json5 +{ + "stores": {}, + "workers": {}, + "schedulers": { + "MAIN_SCHEDULER": { + "simple": {} + }, + } +} +``` + +### Scheduler Options + +The contents of the scheduler type object defines the options. For a list of options see the documentation. See the example below. + +```json5 +{ + "stores": {}, + "workers": {}, + "schedulers": { + "MAIN_SCHEDULER": { + "simple": { + "supported_platform_properties": { + "cpu_count": "minimum", + "memory_kb": "minimum", + "network_kbps": "minimum", + "disk_read_iops": "minimum", + "disk_read_bps": "minimum", + "disk_write_iops": "minimum", + "disk_write_bps": "minimum", + "shm_size": "minimum", + "gpu_count": "minimum", + "gpu_model": "exact", + "cpu_vendor": "exact", + "cpu_arch": "exact", + "cpu_model": "exact", + "kernel_version": "exact", + "OSFamily": "priority", + "container-image": "priority", + } + } + } + }, + "servers": [], + "global": {}, +} +``` + +
+ +
+ Configuring Servers + + ### Servers + +The `servers` configuration object is an array, with two objects, `public`, and `private_workers_servers`. + +```json5 +{ + "stores": {}, + "workers": {}, + "schedulers": {}, + "servers": [{ + "name": "public" + },{ + "name": "private_workers_servers" + }], + "global": {}, +} +``` + + ### Public Server + +The `public` server consists of a `listener` object and a `services` object. The `listener` object is one level of depth and includes an `http` with a `socket address`. The `services` server consists of a `cas`, an `ac`, the `execution`, `capabilities`, and `bytestream`. + +```json5 +{ + "stores": {}, + "workers": {}, + "schedulers": {}, + "servers": [{ + "name": "public", + "listener": { + "http": { + "socket_address": "0.0.0.0:50051" + } + }, + "services": { + "cas": { + "main": { + "cas_store": "WORKER_FAST_SLOW_STORE" + } + }, + "ac": { + "main": { + "ac_store": "AC_MAIN_STORE" + } + }, + "execution": { + "main": { + "cas_store": "WORKER_FAST_SLOW_STORE", + "scheduler": "MAIN_SCHEDULER", + } + }, + "capabilities": { + "main": { + "remote_execution": { + "scheduler": "MAIN_SCHEDULER", + } + } + }, + "bytestream": { + "cas_stores": { + "main": "WORKER_FAST_SLOW_STORE", + } + } + }, + },{ + "name": "private_workers_servers" + }], + "global": {}, +} +``` + + ### Private Server + + > ⚠️ _WARNING_: A private server shouldn't be exposed to the public. ⚠️ + + The `private` server consists of a `listener` object and a `services` object. The `listener` object is one level and includes an `http` with a `socket address`. The `services` server consists of an `experimental_prometheus` object with a `path` field, a `worker_api` object with `scheduler_field`, and an `admin` object. + +```json5 + { + "stores": {}, + "workers": {}, + "schedulers": {}, + "servers": [{ + "name": "public", + "listener": { + "http": { + "socket_address": "0.0.0.0:50051" + } + }, + "services": { + "cas": { + "main": { + "cas_store": "WORKER_FAST_SLOW_STORE" + } + }, + "ac": { + "main": { + "ac_store": "AC_MAIN_STORE" + } + }, + "execution": { + "main": { + "cas_store": "WORKER_FAST_SLOW_STORE", + "scheduler": "MAIN_SCHEDULER", + } + }, + "capabilities": { + "main": { + "remote_execution": { + "scheduler": "MAIN_SCHEDULER", + } + } + }, + "bytestream": { + "cas_stores": { + "main": "WORKER_FAST_SLOW_STORE", + } + } + }, + },{ + "name": "private_workers_servers", + "listener": { + "http": { + "socket_address": "0.0.0.0:50061" + } + }, + "services": { + "experimental_prometheus": { + "path": "/metrics" + }, + // Note: This should be served on a different port, because it has + // a different permission set than the other services. + // In other words, this service is a backend api. The ones above + // are a frontend api. + "worker_api": { + "scheduler": "MAIN_SCHEDULER", + }, + "admin": {}, + "health": {}, + } + }], + "global": {}, +} +``` + +*`global`* is a single-level object and can be added at the end as the configuration object for file descriptors. + +```json5 + "global": { + "max_open_files": 512 + } +``` + +
+ +
+ Complete Example + + Below, you will find a fully tested example that you can also find in [basic_cas.json](https://github.com/TraceMachina/nativelink/blob/main/nativelink-config/examples/basic_cas.json) + +```json5 + +{ + "stores": { + "AC_MAIN_STORE": { + "filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-ac", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-ac", + "eviction_policy": { + // 1gb. + "max_bytes": 1000000000, + } + } + }, + "WORKER_FAST_SLOW_STORE": { + "fast_slow": { + // "fast" must be a "filesystem" store because the worker uses it to make + // hardlinks on disk to a directory where the jobs are running. + "fast": { + "filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", + "eviction_policy": { + // 10gb. + "max_bytes": 10000000000, + } + } + }, + "slow": { + /// Discard data. + /// This example usage has the CAS and the Worker live in the same place, + /// so they share the same underlying CAS. Since workers require a fast_slow + /// store, we use the fast store as our primary data store, and the slow store + /// is just a noop, since there's no shared storage in this config. + "noop": {} + } + } + } + }, + "schedulers": { + "MAIN_SCHEDULER": { + "simple": { + "supported_platform_properties": { + "cpu_count": "minimum", + "memory_kb": "minimum", + "network_kbps": "minimum", + "disk_read_iops": "minimum", + "disk_read_bps": "minimum", + "disk_write_iops": "minimum", + "disk_write_bps": "minimum", + "shm_size": "minimum", + "gpu_count": "minimum", + "gpu_model": "exact", + "cpu_vendor": "exact", + "cpu_arch": "exact", + "cpu_model": "exact", + "kernel_version": "exact", + "OSFamily": "priority", + "container-image": "priority", + // Example of how to set which docker images are available and set + // them in the platform properties. + // "docker_image": "priority", + } + } + } + }, + "workers": [{ + "local": { + "worker_api_endpoint": { + "uri": "grpc://127.0.0.1:50061", + }, + "cas_fast_slow_store": "WORKER_FAST_SLOW_STORE", + "upload_action_result": { + "ac_store": "AC_MAIN_STORE", + }, + "work_directory": "/tmp/nativelink/work", + "platform_properties": { + "cpu_count": { + "values": ["16"], + }, + "memory_kb": { + "values": ["500000"], + }, + "network_kbps": { + "values": ["100000"], + }, + "cpu_arch": { + "values": ["x86_64"], + }, + "OSFamily": { + "values": [""] + }, + "container-image": { + "values": [""] + }, + // Example of how to set which docker images are available and set + // them in the platform properties. + // "docker_image": { + // "query_cmd": "docker images --format {{.Repository}}:{{.Tag}}", + // } + } + } + }], + "servers": [{ + "name": "public", + "listener": { + "http": { + "socket_address": "0.0.0.0:50051" + } + }, + "services": { + "cas": { + "main": { + "cas_store": "WORKER_FAST_SLOW_STORE" + } + }, + "ac": { + "main": { + "ac_store": "AC_MAIN_STORE" + } + }, + "execution": { + "main": { + "cas_store": "WORKER_FAST_SLOW_STORE", + "scheduler": "MAIN_SCHEDULER", + } + }, + "capabilities": { + "main": { + "remote_execution": { + "scheduler": "MAIN_SCHEDULER", + } + } + }, + "bytestream": { + "cas_stores": { + "main": "WORKER_FAST_SLOW_STORE", + } + } + } + }, { + "name": "private_workers_servers", + "listener": { + "http": { + "socket_address": "0.0.0.0:50061" + } + }, + "services": { + "experimental_prometheus": { + "path": "/metrics" + }, + // Note: This should be served on a different port, because it has + // a different permission set than the other services. + // In other words, this service is a backend api. The ones above + // are a frontend api. + "worker_api": { + "scheduler": "MAIN_SCHEDULER", + }, + "admin": {} + } + }], + "global": { + "max_open_files": 512 + } +} +``` + +
diff --git a/nativelink-docs/docs/configuration/Examples.mdx b/nativelink-docs/docs/configuration/Examples.mdx new file mode 100644 index 000000000..b7b44fa32 --- /dev/null +++ b/nativelink-docs/docs/configuration/Examples.mdx @@ -0,0 +1,704 @@ +--- +sidebar_position: 2 +title: 'Config Examples' +description: 'Example configs with explainations' +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +NativeLink uses JSON files for configuration of the CAS and Scheduler. + +This section will break down all of the customizable fields in these JSON files so +they can be modified to suit your needs. + +## Basic CAS + + +The Basic CAS configuration is ideal for running NativeLink locally. +It provides a straightforward setup that is easy to understand and implement, + making it perfect for development and testing environments. + +

+```json5 +{ + "stores": { + "AC_MAIN_STORE": { + "filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-ac", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-ac", + "eviction_policy": { + // 1gb. + "max_bytes": 1000000000, + } + } + }, + "WORKER_FAST_SLOW_STORE": { + "fast_slow": { + // "fast" must be a "filesystem" store because the worker uses it to make + // hardlinks on disk to a directory where the jobs are running. + "fast": { + "filesystem": { + "content_path": "/tmp/nativelink/data-worker-test/content_path-cas", + "temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas", + "eviction_policy": { + // 10gb. + "max_bytes": 10000000000, + } + } + }, + "slow": { + /// Discard data. + /// This example usage has the CAS and the Worker live in the same place, + /// so they share the same underlying CAS. Since workers require a fast_slow + /// store, we use the fast store as our primary data store, and the slow store + /// is just a noop, since there's no shared storage in this config. + "noop": {} + } + } + } + }, + "schedulers": { + "MAIN_SCHEDULER": { + "simple": { + "supported_platform_properties": { + "cpu_count": "minimum", + "memory_kb": "minimum", + "network_kbps": "minimum", + "disk_read_iops": "minimum", + "disk_read_bps": "minimum", + "disk_write_iops": "minimum", + "disk_write_bps": "minimum", + "shm_size": "minimum", + "gpu_count": "minimum", + "gpu_model": "exact", + "cpu_vendor": "exact", + "cpu_arch": "exact", + "cpu_model": "exact", + "kernel_version": "exact", + "OSFamily": "priority", + "container-image": "priority", + // Example of how to set which docker images are available and set + // them in the platform properties. + // "docker_image": "priority", + } + } + } + }, + "workers": [{ + "local": { + "worker_api_endpoint": { + "uri": "grpc://127.0.0.1:50061", + }, + "cas_fast_slow_store": "WORKER_FAST_SLOW_STORE", + "upload_action_result": { + "ac_store": "AC_MAIN_STORE", + }, + "work_directory": "/tmp/nativelink/work", + "platform_properties": { + "cpu_count": { + "values": ["16"], + }, + "memory_kb": { + "values": ["500000"], + }, + "network_kbps": { + "values": ["100000"], + }, + "cpu_arch": { + "values": ["x86_64"], + }, + "OSFamily": { + "values": [""] + }, + "container-image": { + "values": [""] + }, + // Example of how to set which docker images are available and set + // them in the platform properties. + // "docker_image": { + // "query_cmd": "docker images --format {{.Repository}}:{{.Tag}}", + // } + } + } + }], + "servers": [{ + "name": "public", + "listener": { + "http": { + "socket_address": "0.0.0.0:50051" + } + }, + "services": { + "cas": { + "main": { + "cas_store": "WORKER_FAST_SLOW_STORE" + } + }, + "ac": { + "main": { + "ac_store": "AC_MAIN_STORE" + } + }, + "execution": { + "main": { + "cas_store": "WORKER_FAST_SLOW_STORE", + "scheduler": "MAIN_SCHEDULER", + } + }, + "capabilities": { + "main": { + "remote_execution": { + "scheduler": "MAIN_SCHEDULER", + } + } + }, + "bytestream": { + "cas_stores": { + "main": "WORKER_FAST_SLOW_STORE", + } + } + } + }, { + "name": "private_workers_servers", + "listener": { + "http": { + "socket_address": "0.0.0.0:50061" + } + }, + "services": { + "experimental_prometheus": { + "path": "/metrics" + }, + // Note: This should be served on a different port, because it has + // a different permission set than the other services. + // In other words, this service is a backend api. The ones above + // are a frontend api. + "worker_api": { + "scheduler": "MAIN_SCHEDULER", + }, + "admin": {}, + "health": {}, + } + }], + "global": { + "max_open_files": 512 + } +} +``` +

+ +## Filesystem CAS +The Filesystem CAS configuration serves as a robust local alternative to the Basic CAS. + It is designed to store all data on disk, allowing for service restarts without data loss. + This setup is optimized for performance with features like compression and deduplication. + Additionally, it can be used in a cloud environment, providing flexibility + for both local and cloud-based deployments. +

+```json5 +// This configuration will place objects in various folders in +// `/tmp/nativelink/data`. It will store all data on disk and +// allows for restarts of the underlying service. It is optimized +// so objects are compressed, deduplicated and uses some in-memory +// optimizations for certain hot paths. +{ + "stores": { + "FS_CONTENT_STORE": { + "compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-cas", + "temp_path": "/tmp/nativelink/data/tmp_path-cas", + "eviction_policy": { + // 2gb. + "max_bytes": 2000000000, + } + } + } + } + }, + "CAS_MAIN_STORE": { + "verify": { + "backend": { + // Because we are using a dedup store, we can bypass small objects + // and put those objects directly into the content store without + // having to be indexed. This greatly improves performance of serving + // general build content, since many objects are quite small and by + // putting this size distinguish store in place will prevent 1+ index + // read/write per small object request. + "size_partitioning": { + "size": 262144, // 256k. + "lower_store": { + "ref_store": { + "name": "FS_CONTENT_STORE" + } + }, + "upper_store": { + "dedup": { + "index_store": { + // Since our index store is queried so much, we use a fast_slow + // store so it will keep in memory objects that are accessed + // frequently before going to disk. + // Note: indexes are generally quite small, but accessed frequently. + "fast_slow": { + "fast": { + "memory": { + "eviction_policy": { + // 10mb. + "max_bytes": 10000000, + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + // 500mb. + "max_bytes": 500000000, + } + } + } + } + }, + "content_store": { + "ref_store": { + "name": "FS_CONTENT_STORE" + } + } + } + } + } + }, + "verify_size": true, + "hash_verification_function": "sha256" + } + }, + "AC_MAIN_STORE": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-ac", + "temp_path": "/tmp/nativelink/data/tmp_path-ac", + "eviction_policy": { + // 500mb. + "max_bytes": 500000000, + } + } + } + }, + "schedulers": { + "MAIN_SCHEDULER": { + "simple": { + "supported_platform_properties": { + "cpu_count": "minimum", + "memory_kb": "minimum", + "network_kbps": "minimum", + "disk_read_iops": "minimum", + "disk_read_bps": "minimum", + "disk_write_iops": "minimum", + "disk_write_bps": "minimum", + "shm_size": "minimum", + "gpu_count": "minimum", + "gpu_model": "exact", + "cpu_vendor": "exact", + "cpu_arch": "exact", + "cpu_model": "exact", + "kernel_version": "exact", + "docker_image": "priority", + } + } + } + }, + "servers": [{ + "listener": { + "http": { + "socket_address": "0.0.0.0:50051" + } + }, + "services": { + "cas": { + "main": { + "cas_store": "CAS_MAIN_STORE" + } + }, + "ac": { + "main": { + "ac_store": "AC_MAIN_STORE" + } + }, + "execution": { + "main": { + "cas_store": "CAS_MAIN_STORE", + "scheduler": "MAIN_SCHEDULER", + } + }, + "capabilities": { + "main": { + "remote_execution": { + "scheduler": "MAIN_SCHEDULER", + } + } + }, + "bytestream": { + "cas_stores": { + "main": "CAS_MAIN_STORE", + } + } + } + }, { + "listener": { + "http": { + "socket_address": "0.0.0.0:50061" + } + }, + "services": { + // Note: This should be served on a different port, because it has + // a different permission set than the other services. + // In other words, this service is a backend api. The ones above + // are a frontend api. + "worker_api": { + "scheduler": "MAIN_SCHEDULER", + }, + "health": {}, + } + }] +} + +``` +

+ +## S3 Backend with Local Fast CAS + +The S3 Backend configuration is an excellent choice when transitioning to a production-grade + hosted cloud environment. It leverages Amazon S3 for scalable and reliable storage, + combined with a local fast CAS for a simple performant solution. + +

+```json5 +{ + "stores": { + "CAS_MAIN_STORE": { + "verify": { + "backend": { + "dedup": { + "index_store": { + "fast_slow": { + "fast": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + // 500mb. + "max_bytes": 500000000, + } + } + }, + "slow": { + "experimental_s3_store": { + "region": "eu-north-1", + "bucket": "crossplane-bucket-af79aeca9", + "key_prefix": "test-prefix-index/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5, + }, + "additional_max_concurrent_requests": 10 + } + } + } + }, + "content_store": { + "compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "fast_slow": { + "fast": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-content", + "temp_path": "/tmp/nativelink/data/tmp_path-content", + "eviction_policy": { + // 2gb. + "max_bytes": 2000000000, + } + } + }, + "slow": { + "experimental_s3_store": { + "region": "eu-north-1", + "bucket": "crossplane-bucket-af79aeca9", + "key_prefix": "test-prefix-dedup-cas/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5, + }, + "additional_max_concurrent_requests": 10 + } + } + } + } + } + } + } + }, + "verify_size": true, + "hash_verification_function": "sha256" + } + }, + "AC_MAIN_STORE": { + "fast_slow": { + "fast": { + "memory": { + "eviction_policy": { + // 100mb. + "max_bytes": 100000000, + } + }, + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-ac", + "temp_path": "/tmp/nativelink/data/tmp_path-ac", + "eviction_policy": { + // 500mb. + "max_bytes": 500000000, + } + } + }, + "slow": { + "experimental_s3_store": { + "region": "eu-north-1", + "bucket": "crossplane-bucket-af79aeca9", + "key_prefix": "test-prefix-ac/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5, + }, + "additional_max_concurrent_requests": 10 + } + } + } + } + }, + "schedulers": { + "MAIN_SCHEDULER": { + "simple": { + "supported_platform_properties": { + "cpu_count": "minimum", + "memory_kb": "minimum", + "network_kbps": "minimum", + "disk_read_iops": "minimum", + "disk_read_bps": "minimum", + "disk_write_iops": "minimum", + "disk_write_bps": "minimum", + "shm_size": "minimum", + "gpu_count": "minimum", + "gpu_model": "exact", + "cpu_vendor": "exact", + "cpu_arch": "exact", + "cpu_model": "exact", + "kernel_version": "exact", + "docker_image": "priority", + } + } + } + }, + "servers": [{ + "listener": { + "http": { + "socket_address": "0.0.0.0:50051" + } + }, + "services": { + "cas": { + "main": { + "cas_store": "CAS_MAIN_STORE" + } + }, + "ac": { + "main": { + "ac_store": "AC_MAIN_STORE" + } + }, + "execution": { + "main": { + "cas_store": "CAS_MAIN_STORE", + "scheduler": "MAIN_SCHEDULER", + } + }, + "capabilities": { + "main": { + "remote_execution": { + "scheduler": "MAIN_SCHEDULER", + } + } + }, + "bytestream": { + "cas_stores": { + "main": "CAS_MAIN_STORE", + } + }, + "health": {}, + } + }] +} + +``` +

+ +## Redis CAS + +The Redis CAS configuration is designed for high-performance cloud environments. + It uses Redis as the fast-end storage, providing quick access to frequently used data. + This configuration can be effectively paired with S3 for the slow store, + creating a highly efficient and scalable production setup. + It is particularly well-suited for use with Kubernetes, + enabling seamless integration and management of cloud resources. + +

+```json5 +{ + "stores": { + "CAS_FAST_SLOW_STORE": { + "fast_slow": { + "fast": { + "redis_store": { + "addresses": [ + "redis://127.0.0.1:6379/", // Master node 1 + "redis://127.0.0.1:6380/", // Master node 2 + "redis://127.0.0.1:6381/", // Master node 3 + "redis://127.0.0.1:6382/", // Master node 3 + "redis://127.0.0.1:6383/", // Master node 3 + "redis://127.0.0.1:6384/" // Master node 3 + //"redis://172.18.0.2:6379/" // Master node 3 + // "redis://172.18.0.3:6379/", // Master node 3 + // "redis://172.18.0.4:6379/" // Master node 3 + ] + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": 120000000000 + } + } + } + } + }, + "AC_FAST_SLOW_STORE": { + "fast_slow": { + "fast": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-index", + "temp_path": "/tmp/nativelink/data/tmp_path-index", + "eviction_policy": { + "max_bytes": 120000000000 + } + } + }, + "slow": { + "filesystem": { + "content_path": "/tmp/nativelink/data/content_path-ac", + "temp_path": "/tmp/nativelink/data/tmp_path-ac", + "eviction_policy": { + "max_bytes": 5000000000 + } + } + } + } + }, + "AC_MAIN_STORE": { + "completeness_checking": { + "backend": { + "ref_store": { + "name": "AC_FAST_SLOW_STORE" + } + }, + "cas_store": { + "ref_store": { + "name": "CAS_MAIN_STORE" + } + } + } + }, + "CAS_MAIN_STORE": { + "existence_cache": { + "backend": { + "compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "ref_store": { + "name": "CAS_FAST_SLOW_STORE" + } + } + } + } + } + } + }, + "servers": [ + { + "listener": { + "http": { + "socket_address": "0.0.0.0:50051" + } + }, + "services": { + "cas": { + "main": { + "cas_store": "CAS_MAIN_STORE" + } + }, + "ac": { + "main": { + "ac_store": "AC_MAIN_STORE" + } + }, + "capabilities": {}, + "bytestream": { + "cas_stores": { + "main": "CAS_MAIN_STORE" + } + } + } + }, + { + "listener": { + "http": { + "socket_address": "0.0.0.0:50061" + } + }, + "services": { + "experimental_prometheus": { + "path": "/metrics" + } + } + } + ] + } +``` +

diff --git a/nativelink-docs/docs/configuration/Introduction.mdx b/nativelink-docs/docs/configuration/Introduction.mdx new file mode 100644 index 000000000..0c3240196 --- /dev/null +++ b/nativelink-docs/docs/configuration/Introduction.mdx @@ -0,0 +1,265 @@ +--- +sidebar_position: 1 +title: 'Introduction' +description: 'Learn how to configure NativeLink' +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +NativeLink uses JSON files for configuration of the CAS and Scheduler. + +This section will break down all of the customizable fields in these JSON files so +they can be modified to suit your needs. + +## ⚙️ Getting Started Configuring NativeLink + + + + ### 1. Copy the [Sample CAS.json](https://github.com/TraceMachina/nativelink/blob/main/deployment-examples/kubernetes/cas.json). +

+ + ```json5 + { + "stores": { + "AC_FAST_SLOW_STORE": { + "fast_slow": { + "fast": { + "memory": { + "eviction_policy": { + "max_bytes": 1000000000 + } + } + }, + "slow": { + "experimental_s3_store": { + "region": "${NATIVE_LINK_AWS_REGION:-us-east-1}", + "bucket": "${NATIVE_LINK_AWS_S3_CAS_BUCKET:-not_set}", + "key_prefix": "ac/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "additional_max_concurrent_requests": 10 + } + } + } + }, + "AC_S3_STORE": { + "completeness_checking": { + "backend": { + "ref_store": { + "name": "AC_FAST_SLOW_STORE" + } + }, + "cas_store": { + "ref_store": { + "name": "CAS_S3_STORE" + } + } + } + }, + "CAS_FAST_SLOW_STORE": { + "fast_slow": { + "fast": { + "memory": { + "eviction_policy": { + "max_bytes": 1000000000 + } + } + }, + "slow": { + "experimental_s3_store": { + "region": "${NATIVE_LINK_AWS_REGION:-us-east-1}", + "bucket": "${NATIVE_LINK_AWS_S3_CAS_BUCKET:-not_set}", + "key_prefix": "cas/", + "retry": { + "max_retries": 6, + "delay": 0.3, + "jitter": 0.5 + }, + "additional_max_concurrent_requests": 10 + } + } + } + }, + "CAS_S3_STORE": { + "existence_cache": { + "backend": { + "compression": { + "compression_algorithm": { + "lz4": {} + }, + "backend": { + "ref_store": { + "name": "CAS_FAST_SLOW_STORE" + } + } + } + } + } + } + }, + "servers": [ + { + "listener": { + "http": { + "socket_address": "0.0.0.0:{{ .Values.cas.service.grpcPort }}" + } + }, + "services": { + "cas": { + "main": { + "cas_store": "CAS_S3_STORE" + } + }, + "ac": { + "main": { + "ac_store": "AC_S3_STORE" + } + }, + "capabilities": {}, + "bytestream": { + "cas_stores": { + "main": "CAS_S3_STORE" + } + } + } + }, + { + "listener": { + "http": { + "socket_address": "0.0.0.0:50061" + } + }, + "services": { + "experimental_prometheus": { + "path": "/metrics" + } + } + } + ] + } + ``` + +

+ + + + + + ### 1. Copy the [Sample Scheduler.json](https://github.com/TraceMachina/nativelink/blob/main/deployment-examples/kubernetes/scheduler.json). +

+ + ```json5 + { + "stores": { + "GRPC_LOCAL_STORE": { + // Note: This file is used to test GRPC store. + "grpc": { + "instance_name": "main", + "endpoints": [ + {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} + ], + "store_type": "cas" + } + }, + "GRPC_LOCAL_AC_STORE": { + // Note: This file is used to test GRPC store. + "grpc": { + "instance_name": "main", + "endpoints": [ + {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} + ], + "store_type": "ac" + } + } + }, + "schedulers": { + "MAIN_SCHEDULER": { + "simple": { + "supported_platform_properties": { + "cpu_count": "minimum", + "OSFamily": "exact", + "container-image": "exact" + } + } + } + }, + "servers": [{ + "listener": { + "http": { + "socket_address": "0.0.0.0:50052" + } + }, + "services": { + "ac": { + "main": { + "ac_store": "GRPC_LOCAL_AC_STORE" + } + }, + "execution": { + "main": { + "cas_store": "GRPC_LOCAL_STORE", + "scheduler": "MAIN_SCHEDULER", + } + }, + "capabilities": { + "main": { + "remote_execution": { + "scheduler": "MAIN_SCHEDULER", + } + } + } + } + }, + { + "listener": { + "http": { + "socket_address": "0.0.0.0:50061", + } + }, + "services": { + // Note: This should be served on a different port, because it has + // a different permission set than the other services. + // In other words, this service is a backend api. The ones above + // are a frontend api. + "worker_api": { + "scheduler": "MAIN_SCHEDULER", + } + } + }] +} + ``` + +

+ + + + + + ### 2. Paste default sample where needed in custom [deployment](https://github.com/TraceMachina/nativelink/tree/main/deployment-examples). + ### 3. Change settings to preferred values. + +:::note + +We recommend that you leave most of the values in place and start by only + modifying the values under `stores` such as `AC_FAST_SLOW_STORE`, `CAS_FAST_SLOW_STORE`, or others in the example below before trying out changes to more advanced + settings like the scheduler. + +::: + +### Please refer to the [Configuration Reference Page](/configuration/Reference) to see additional options. + +### For more advanced configuration, please refer to the [instructions for assembling a configuration from scratch](./Assembling). + +Additional examples for store configurations and additional store options can be found in the **[store configuration documentation](https://github.com/TraceMachina/nativelink/blob/main/nativelink-config/src/stores.rs)**. diff --git a/nativelink-docs/docs/Configuration.mdx b/nativelink-docs/docs/configuration/Reference.mdx similarity index 62% rename from nativelink-docs/docs/Configuration.mdx rename to nativelink-docs/docs/configuration/Reference.mdx index 354ad3716..b451db758 100644 --- a/nativelink-docs/docs/Configuration.mdx +++ b/nativelink-docs/docs/configuration/Reference.mdx @@ -1,254 +1,13 @@ --- -sidebar_position: 3 -title: 'Configuration' -description: 'Learn how to configure NativeLink' +sidebar_position: 2 +title: 'Reference' +description: 'Reference for configuration fields' --- import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -NativeLink uses JSON files for configuration of the CAS and Scheduler. - -This section will break down all of the customizable fields in these JSON files so -they can be modified to suit your needs. - -## ⚙️ Configuration Instructions - - - - ### 1. Copy the [Sample CAS.json](https://github.com/TraceMachina/nativelink/blob/main/deployment-examples/kubernetes/cas.json). -

- - ```json5 - { - "stores": { - "AC_FAST_SLOW_STORE": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": 1000000000 - } - } - }, - "slow": { - "experimental_s3_store": { - "region": "${NATIVE_LINK_AWS_REGION:-us-east-1}", - "bucket": "${NATIVE_LINK_AWS_S3_CAS_BUCKET:-not_set}", - "key_prefix": "ac/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "additional_max_concurrent_requests": 10 - } - } - } - }, - "AC_S3_STORE": { - "completeness_checking": { - "backend": { - "ref_store": { - "name": "AC_FAST_SLOW_STORE" - } - }, - "cas_store": { - "ref_store": { - "name": "CAS_S3_STORE" - } - } - } - }, - "CAS_FAST_SLOW_STORE": { - "fast_slow": { - "fast": { - "memory": { - "eviction_policy": { - "max_bytes": 1000000000 - } - } - }, - "slow": { - "experimental_s3_store": { - "region": "${NATIVE_LINK_AWS_REGION:-us-east-1}", - "bucket": "${NATIVE_LINK_AWS_S3_CAS_BUCKET:-not_set}", - "key_prefix": "cas/", - "retry": { - "max_retries": 6, - "delay": 0.3, - "jitter": 0.5 - }, - "additional_max_concurrent_requests": 10 - } - } - } - }, - "CAS_S3_STORE": { - "existence_cache": { - "backend": { - "compression": { - "compression_algorithm": { - "lz4": {} - }, - "backend": { - "ref_store": { - "name": "CAS_FAST_SLOW_STORE" - } - } - } - } - } - } - }, - "servers": [ - { - "listener": { - "http": { - "socket_address": "0.0.0.0:{{ .Values.cas.service.grpcPort }}" - } - }, - "services": { - "cas": { - "main": { - "cas_store": "CAS_S3_STORE" - } - }, - "ac": { - "main": { - "ac_store": "AC_S3_STORE" - } - }, - "capabilities": {}, - "bytestream": { - "cas_stores": { - "main": "CAS_S3_STORE" - } - } - } - }, - { - "listener": { - "http": { - "socket_address": "0.0.0.0:50061" - } - }, - "services": { - "experimental_prometheus": { - "path": "/metrics" - } - } - } - ] - } - ``` - -

- - - - - - ### 1. Copy the [Sample Scheduler.json](https://github.com/TraceMachina/nativelink/blob/main/deployment-examples/kubernetes/scheduler.json). -

- - ```json5 - { - "stores": { - "GRPC_LOCAL_STORE": { - // Note: This file is used to test GRPC store. - "grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "store_type": "cas" - } - }, - "GRPC_LOCAL_AC_STORE": { - // Note: This file is used to test GRPC store. - "grpc": { - "instance_name": "main", - "endpoints": [ - {"address": "grpc://${CAS_ENDPOINT:-127.0.0.1}:50051"} - ], - "store_type": "ac" - } - } - }, - "schedulers": { - "MAIN_SCHEDULER": { - "simple": { - "supported_platform_properties": { - "cpu_count": "minimum", - "OSFamily": "exact", - "container-image": "exact" - } - } - } - }, - "servers": [{ - "listener": { - "http": { - "socket_address": "0.0.0.0:50052" - } - }, - "services": { - "ac": { - "main": { - "ac_store": "GRPC_LOCAL_AC_STORE" - } - }, - "execution": { - "main": { - "cas_store": "GRPC_LOCAL_STORE", - "scheduler": "MAIN_SCHEDULER", - } - }, - "capabilities": { - "main": { - "remote_execution": { - "scheduler": "MAIN_SCHEDULER", - } - } - } - } - }, - { - "listener": { - "http": { - "socket_address": "0.0.0.0:50061", - } - }, - "services": { - // Note: This should be served on a different port, because it has - // a different permission set than the other services. - // In other words, this service is a backend api. The ones above - // are a frontend api. - "worker_api": { - "scheduler": "MAIN_SCHEDULER", - } - } - }] -} - ``` - -

- - - - - - ### 2. Paste default sample where needed in custom [deployment](https://github.com/TraceMachina/nativelink/tree/main/deployment-examples). - ### 3. Change settings to preferred values. +This reference details the customizable fields in the JSON configuration files used by NativeLink for the CAS and Scheduler. @@ -315,5 +74,3 @@ they can be modified to suit your needs. | servers | Array | Defines the server configurations.| `{"servers": {...}}`| - -Additional examples for store configurations and additional store options can be found in the **[store configuration documentation](https://github.com/TraceMachina/nativelink/blob/main/nativelink-config/src/stores.rs)**. diff --git a/nativelink-docs/docs/configuration/_category_.json b/nativelink-docs/docs/configuration/_category_.json new file mode 100644 index 000000000..323ea7767 --- /dev/null +++ b/nativelink-docs/docs/configuration/_category_.json @@ -0,0 +1,10 @@ +{ + "label": "Configuration", + "position": 2, + "collapsed": false, + "collapsible": true, + "link": { + "type": "generated-index", + "description": "Reference docs for help with configuration your NativeLink." + } +} diff --git a/nativelink-docs/docs/deployment-examples/_category_.json b/nativelink-docs/docs/deployment-examples/_category_.json index 012a388f4..09bd4117d 100644 --- a/nativelink-docs/docs/deployment-examples/_category_.json +++ b/nativelink-docs/docs/deployment-examples/_category_.json @@ -1,6 +1,8 @@ { "label": "Deployment Examples", - "position": 2, + "position": 3, + "collapsible": true, + "collapsed": false, "link": { "type": "generated-index", "description": "See the capabilities of remote execution and cache." diff --git a/nativelink-docs/sidebars.js b/nativelink-docs/sidebars.js index 332758032..5bb60a07d 100644 --- a/nativelink-docs/sidebars.js +++ b/nativelink-docs/sidebars.js @@ -16,18 +16,6 @@ const sidebars = { // By default, Docusaurus generates a sidebar from the docs folder structure tutorialSidebar: [{type: 'autogenerated', dirName: '.'}], - // But you can create a sidebar manually - /* - tutorialSidebar: [ - 'intro', - 'hello', - { - type: 'category', - label: 'Tutorial', - items: ['tutorial-basics/create-a-document'], - }, - ], - */ }; export default sidebars;