Skip to content

Commit

Permalink
add zstd nightly runs for nyc_taxis and pmc (#4208)
Browse files Browse the repository at this point in the history 
Signed-off-by: Rishabh Singh <[email protected]>
  • Loading branch information
@rishabh6788
rishabh6788 authored Nov 11, 2023
1 parent efe614c commit 487bd2e
Show file tree
Hide file tree
Showing 4 changed files with 20 additions and 8 deletions.
7 changes: 5 additions & 2 deletions jenkins/opensearch/benchmark-test.jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ pipeline {
H 3 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.12.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=so;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"}

H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3"}
H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3"}
H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=pmc;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3"}

H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica-zstd-codec;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3","index_settings":{"codec":"zstd"}}
H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica-zstd-codec;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3","index_settings":{"codec":"zstd"}
H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=pmc;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica-zstd-codec;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3","index_settings":{"codec":"zstd"}}
Expand Down Expand Up @@ -87,8 +90,8 @@ pipeline {
'''
+
'''
H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica-logbyte;ADDITIONAL_CONFIG=indices.time_series_index.default_index_merge_policy:log_byte_size;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3","bulk_indexing_clients":"1","index_settings":{"index.merge.log_byte_size_policy.merge_factor":"32","index.merge.log_byte_size_policy.max_merge_segment":"500MB","refresh_interval":"1s"}}
H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica-tiered;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3","bulk_indexing_clients":"1","index_settings":{"index.merge.policy.max_merged_segment":"500MB","refresh_interval":"1s"}}
H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica-logbyte;ADDITIONAL_CONFIG=indices.time_series_index.default_index_merge_policy:log_byte_size;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3","bulk_indexing_clients":"1","index_settings":{"index.merge.log_byte_size_policy.merge_factor":"32","index.merge.log_byte_size_policy.max_merge_segment":"500MB","refresh_interval":"1s"}}
H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica-tiered;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3","bulk_indexing_clients":"1","index_settings":{"index.merge.policy.max_merged_segment":"500MB","refresh_interval":"1s"}}

H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-concurrent-search-0-slice-3-data-3-shards-0-replica-snapshot;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"http_logs_default"};TEST_PROCEDURE=restore-from-snapshot
H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-concurrent-search-0-slice-3-data-3-shards-0-replica-snapshot;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_default"};TEST_PROCEDURE=restore-from-snapshot
Expand Down
7 changes: 5 additions & 2 deletions ...ns/jenkinsjob-regression-files/opensearch/benchmark-test-distribution-url.jenkinsfile.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@
H 3 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.12.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=so;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"}

H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3"}
H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3"}
H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=pmc;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3"}

H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica-zstd-codec;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3","index_settings":{"codec":"zstd"}}
H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica-zstd-codec;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3","index_settings":{"codec":"zstd"}
H 4 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=pmc;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica-zstd-codec;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3","index_settings":{"codec":"zstd"}}
Expand Down Expand Up @@ -65,8 +68,8 @@
H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"}
H 13 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/3.0.0/latest/linux/x64/tar/dist/opensearch/manifest.yml;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;CAPTURE_SEGMENT_REPLICATION_STAT=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:3x,cluster-config:x64-r5.xlarge-3-data-3-shards-1-replica;ADDITIONAL_CONFIG=cluster.indices.replication.strategy:SEGMENT;WORKLOAD_PARAMS={"number_of_replicas":"1","number_of_shards":"3"}

H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica-logbyte;ADDITIONAL_CONFIG=indices.time_series_index.default_index_merge_policy:log_byte_size;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3","bulk_indexing_clients":"1","index_settings":{"index.merge.log_byte_size_policy.merge_factor":"32","index.merge.log_byte_size_policy.max_merge_segment":"500MB","refresh_interval":"1s"}}
H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:enabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica-tiered;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3","bulk_indexing_clients":"1","index_settings":{"index.merge.policy.max_merged_segment":"500MB","refresh_interval":"1s"}}
H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica-logbyte;ADDITIONAL_CONFIG=indices.time_series_index.default_index_merge_policy:log_byte_size;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3","bulk_indexing_clients":"1","index_settings":{"index.merge.log_byte_size_policy.merge_factor":"32","index.merge.log_byte_size_policy.max_merge_segment":"500MB","refresh_interval":"1s"}}
H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-3-data-3-shards-0-replica-tiered;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"3","bulk_indexing_clients":"1","index_settings":{"index.merge.policy.max_merged_segment":"500MB","refresh_interval":"1s"}}

H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-concurrent-search-0-slice-3-data-3-shards-0-replica-snapshot;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"http_logs_default"};TEST_PROCEDURE=restore-from-snapshot
H 14 * * * %BUNDLE_MANIFEST_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.11.0/8649/linux/x64/tar/dist/opensearch/manifest.yml;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=false;DATA_NODE_COUNT=3;USE_50_PERCENT_HEAP=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-concurrent-search-0-slice-3-data-3-shards-0-replica-snapshot;ADDITIONAL_CONFIG=opensearch.experimental.feature.concurrent_segment_search.enabled:true search.concurrent_segment_search.enabled:true search.concurrent.max_slice_count:0;WORKLOAD_PARAMS={"snapshot_repo_name":"benchmark-workloads-repo","snapshot_bucket_name":"benchmark-workload-snapshots","snapshot_region":"us-east-1","snapshot_base_path":"workload-snapshots","snapshot_name":"nyc_taxis_default"};TEST_PROCEDURE=restore-from-snapshot
Expand Down
Loading

0 comments on commit 487bd2e

Please sign in to comment.