Add loadtest environment CDK stack

scripts/loadtest-environment/.gitignore

@@ -0,0 +1,8 @@
+.idea
+*.js
+!jest.config.js
+*.d.ts
+cdk.context.json
+cdk.out
+node_modules
+package-lock.json

scripts/loadtest-environment/.prettierrc

@@ -0,0 +1,7 @@
+{
+  "trailingComma": "all",
+  "tabWidth": 2,
+  "semi": true,
+  "arrowParens": "always",
+  "printWidth": 120
+}

scripts/loadtest-environment/LICENSE

@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) 2024 - Restate Software, Inc., Restate GmbH
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE

scripts/loadtest-environment/README.md

@@ -0,0 +1,13 @@
+# Restate Load Test Sandbox
+
+This CDK project creates reproducible environments for testing Restate performance.
+
+You will need an AWS account, bootstrapped for CDK use, and npm. Run `npm install` to fetch the required dependencies.
+
+Before you deploy the stack, you may want to customize the instance type and storage configuration in `bin/loadtest-env.ts`.
+
+Run `npx cdk deploy` to create the EC2 instance. Currently this stack does not automatically download the Restate server, it only sets up the environment for building it from source - or running a prebuilt release.
+
+To connect to your EC2 instance, use `aws ssm start-session --target <instance-id>`. You can find some predefined load tests in the `tests` directory. You can find some standard restate-server configurations in the `config` directory.
+
+If you want to temporarily suspend the machine, you can do so with `aws ec2 stop-instances --instance-ids <instance-id>` and later re-start it. You will still pay for the associated EBS volumes even while the instance is stopped. Note that the contents of any local NVMe devices will be wiped if you do so. Tear down the stack using `npx cdk destroy` when you are done with it.

scripts/loadtest-environment/bin/loadtest-env.ts

@@ -0,0 +1,58 @@
+#!/usr/bin/env node
+
+/*
+ * Copyright (c) 2024 - Restate Software, Inc., Restate GmbH
+ *
+ * This file is part of the Restate load test environment,
+ * which is released under the MIT license.
+ *
+ * You can find a copy of the license in file LICENSE in the
+ * scripts/loadtest-environment directory of this repository, or at
+ * https://github.com/restatedev/retate/blob/main/scripts/loadtest-environment/LICENSE
+ */
+
+import "source-map-support/register";
+import * as cdk from "aws-cdk-lib";
+import * as ec2 from "aws-cdk-lib/aws-ec2";
+import { LoadTestEnvironmentStack } from "../lib/loadtest-environment-stack";
+
+// Graviton2 arm64, 8vCPU x 32GB RAM
+const INSTANCE_TYPE_MID = ec2.InstanceType.of(ec2.InstanceClass.M6G, ec2.InstanceSize.XLARGE2);
+
+// Xeon x86_64, 32vCPU x64GB RAM, 1.9TB local NVMe SSD
+const INSTANCE_TYPE_HIGH = ec2.InstanceType.of(ec2.InstanceClass.C6ID, ec2.InstanceSize.XLARGE8);
+
+const EBS_VOLUME_MID = {
+  volumeType: ec2.EbsDeviceVolumeType.GP3,
+  volumeSize: 16, // GiB
+  deleteOnTermination: true,
+  iops: 3_000,
+  throughput: 250, // MiB/s
+};
+
+const EBS_VOLUME_HIGH = {
+  volumeType: ec2.EbsDeviceVolumeType.GP3,
+  volumeSize: 64, // GiB
+  deleteOnTermination: true,
+  iops: 16_000,
+  throughput: 1_000, // MiB/s
+};
+
+const EBS_VOLUME_ULTRA = {
+  volumeType: ec2.EbsDeviceVolumeType.IO2,
+  volumeSize: 64, // GiB
+  deleteOnTermination: true,
+  iops: 64_000,
+};
+
+const app = new cdk.App();
+
+new LoadTestEnvironmentStack(app, `restate-benchmark-sandbox-${process.env.USER}`, {
+  env: {
+    account: process.env.CDK_DEFAULT_ACCOUNT,
+    region: process.env.CDK_DEFAULT_REGION,
+  },
+  instanceType: INSTANCE_TYPE_MID,
+  vpcId: undefined, // use default VPC unless specified
+  ebsVolume: EBS_VOLUME_MID, // optional EBS volume
+});

scripts/loadtest-environment/cdk.json

@@ -0,0 +1,67 @@
+{
+  "app": "npx ts-node --prefer-ts-exts bin/loadtest-env.ts",
+  "watch": {
+    "include": ["**"],
+    "exclude": [
+      "README.md",
+      "cdk*.json",
+      "**/*.d.ts",
+      "**/*.js",
+      "tsconfig.json",
+      "package*.json",
+      "yarn.lock",
+      "node_modules",
+      "test"
+    ]
+  },
+  "context": {
+    "@aws-cdk/aws-lambda:recognizeLayerVersion": true,
+    "@aws-cdk/core:checkSecretUsage": true,
+    "@aws-cdk/core:target-partitions": ["aws", "aws-cn"],
+    "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
+    "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true,
+    "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true,
+    "@aws-cdk/aws-iam:minimizePolicies": true,
+    "@aws-cdk/core:validateSnapshotRemovalPolicy": true,
+    "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true,
+    "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true,
+    "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true,
+    "@aws-cdk/aws-apigateway:disableCloudWatchRole": true,
+    "@aws-cdk/core:enablePartitionLiterals": true,
+    "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true,
+    "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true,
+    "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true,
+    "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true,
+    "@aws-cdk/aws-route53-patters:useCertificate": true,
+    "@aws-cdk/customresources:installLatestAwsSdkDefault": false,
+    "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true,
+    "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true,
+    "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true,
+    "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true,
+    "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true,
+    "@aws-cdk/aws-redshift:columnId": true,
+    "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true,
+    "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true,
+    "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true,
+    "@aws-cdk/aws-kms:aliasNameRef": true,
+    "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true,
+    "@aws-cdk/core:includePrefixInUniqueNameGeneration": true,
+    "@aws-cdk/aws-efs:denyAnonymousAccess": true,
+    "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true,
+    "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true,
+    "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true,
+    "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true,
+    "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true,
+    "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true,
+    "@aws-cdk/aws-codepipeline-actions:useNewDefaultBranchForCodeCommitSource": true,
+    "@aws-cdk/aws-cloudwatch-actions:changeLambdaPermissionLogicalIdForLambdaAction": true,
+    "@aws-cdk/aws-codepipeline:crossAccountKeysDefaultValueToFalse": true,
+    "@aws-cdk/aws-codepipeline:defaultPipelineTypeToV2": true,
+    "@aws-cdk/aws-kms:reduceCrossAccountRegionPolicyScope": true,
+    "@aws-cdk/aws-eks:nodegroupNameAttribute": true,
+    "@aws-cdk/aws-ec2:ebsDefaultGp3Volume": true,
+    "@aws-cdk/aws-ecs:removeDefaultDeploymentAlarm": true,
+    "@aws-cdk/custom-resources:logApiResponseDataPropertyTrueDefault": false,
+    "@aws-cdk/aws-s3:keepNotificationInImportedBucket": false
+  }
+}

scripts/loadtest-environment/config/restate-1.1.0.toml

@@ -0,0 +1,159 @@
+roles = ["worker", "admin", "metadata-store"]
+cluster-name = "perftest"
+allow-bootstrap = true
+bind-address = "0.0.0.0:5122"
+advertised-address = "http://127.0.0.1:5122/"
+bootstrap-num-partitions = 24
+shutdown-timeout = "1m"
+tracing-filter = "info"
+log-filter = "warn,restate=warn"
+log-format = "pretty"
+log-disable-ansi-codes = false
+connect-timeout = "10s"
+disable-prometheus = false
+rocksdb-total-memory-size = "32.0 GB"
+rocksdb-total-memtables-ratio = 0.5
+rocksdb-high-priority-bg-threads = 2
+rocksdb-write-stall-threshold = "3s"
+rocksdb-enable-stall-on-memory-limit = false
+rocksdb-perf-level = "enable-count"
+metadata-update-interval = "3s"
+
+[metadata-store-client]
+type = "embedded"
+address = "http://127.0.0.1:5123/"
+
+[metadata-store-client-backoff-policy]
+type = "exponential"
+initial-interval = "10ms"
+factor = 2.0
+max-interval = "100ms"
+
+[tracing-headers]
+
+[http-keep-alive-options]
+interval = "40s"
+timeout = "20s"
+
+[network-error-retry-policy]
+type = "exponential"
+initial-interval = "10ms"
+factor = 2.0
+max-attempts = 15
+max-interval = "5s"
+
+[worker]
+internal-queue-length = 1000
+cleanup-interval = "1h"
+experimental-feature-new-invocation-status-table = false
+max-command-batch-size = 4
+
+[worker.storage]
+rocksdb-disable-direct-io-for-reads = false
+rocksdb-disable-direct-io-for-flush-and-compactions = false
+rocksdb-disable-wal = true
+rocksdb-disable-statistics = false
+rocksdb-max-background-jobs = 10
+rocksdb-compaction-readahead-size = "2.0 MB"
+rocksdb-statistics-level = "except-timers"
+num-partitions-to-share-memory-budget = 24
+rocksdb-memory-budget = "1.4 GB"
+rocksdb-memory-ratio = 0.49000000953674316
+persist-lsn-interval = "1h"
+persist-lsn-threshold = 1000
+
+[worker.invoker]
+inactivity-timeout = "1m"
+abort-timeout = "1m"
+message-size-warning = "10.0 MB"
+in-memory-queue-length-limit = 1056784
+concurrent-invocations-limit = 100
+
+[worker.invoker.retry-policy]
+type = "exponential"
+initial-interval = "50ms"
+factor = 2.0
+max-interval = "10s"
+
+[admin]
+bind-address = "0.0.0.0:9070"
+heartbeat-interval = "1s 500ms"
+log-trim-interval = "1h"
+log-trim-threshold = 1000
+default-replication-strategy = "on-all-nodes"
+
+[admin.query-engine]
+memory-size = "4.0 GB"
+pgsql-bind-address = "0.0.0.0:9071"
+
+[ingress]
+bind-address = "0.0.0.0:8080"
+kafka-clusters = []
+
+[bifrost]
+default-provider = "local"
+seal-retry-interval = "2s"
+append-retry-min-interval = "10ms"
+append-retry-max-interval = "1s"
+
+[bifrost.local]
+rocksdb-disable-direct-io-for-reads = false
+rocksdb-disable-direct-io-for-flush-and-compactions = false
+rocksdb-disable-wal = false
+rocksdb-disable-statistics = false
+rocksdb-max-background-jobs = 10
+rocksdb-compaction-readahead-size = "2.0 MB"
+rocksdb-statistics-level = "except-timers"
+rocksdb-memory-budget = "1.4 GB"
+rocksdb-memory-ratio = 0.5
+rocksdb-disable-wal-fsync = false
+writer-batch-commit-count = 5000
+writer-batch-commit-duration = "0s"
+
+[bifrost.replicated-loglet]
+
+[bifrost.read-retry-policy]
+type = "exponential"
+initial-interval = "50ms"
+factor = 2.0
+max-attempts = 50
+max-interval = "1s"
+
+[metadata-store]
+bind-address = "0.0.0.0:5123"
+request-queue-length = 32
+rocksdb-memory-budget = "28.5 MB"
+rocksdb-memory-ratio = 0.009999999776482582
+
+[metadata-store.rocksdb]
+rocksdb-disable-direct-io-for-reads = false
+rocksdb-disable-direct-io-for-flush-and-compactions = false
+rocksdb-disable-wal = false
+rocksdb-disable-statistics = false
+rocksdb-max-background-jobs = 10
+rocksdb-compaction-readahead-size = "2.0 MB"
+rocksdb-statistics-level = "except-timers"
+
+[networking]
+handshake-timeout = "3s"
+
+[networking.connect-retry-policy]
+type = "exponential"
+initial-interval = "10ms"
+factor = 2.0
+max-attempts = 10
+max-interval = "500ms"
+
+[log-server]
+rocksdb-disable-direct-io-for-reads = false
+rocksdb-disable-direct-io-for-flush-and-compactions = false
+rocksdb-disable-wal = false
+rocksdb-disable-statistics = false
+rocksdb-max-background-jobs = 10
+rocksdb-compaction-readahead-size = "2.0 MB"
+rocksdb-statistics-level = "except-timers"
+rocksdb-memory-budget = "1.4 GB"
+rocksdb-memory-ratio = 0.5
+rocksdb-disable-wal-fsync = false
+writer-batch-commit-count = 5000
+incoming-network-queue-length = 1000

scripts/loadtest-environment/docs/benchmarking.md

@@ -0,0 +1,6 @@
+# Benchmarking Restate Server
+
+- Launch a Restate server. You can build it from source with `cargo run --release --bin restate-server ...` or run a prebuilt binary using `npx @restatedev/restate-server --config-file .../restate-1.1.0-performance.toml --base-dir ../storage`
+- Start the mock Counter service available from the Restate source tree: `cargo run --release -p mock-service-endpoint --bin mock-service-endpoint`.
+- Register the service: `restate deployments register http://localhost:9080 --yes`
+- Run one of the provided wrk scripts, e.g. `run-test-400c-5m.sh`