Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SOLR-8393: Component for resource usage planning #1638

Draft
wants to merge 18 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
2122a0e
SOLR-8393: Component for resource usage planning
igiguere May 10, 2023
107d383
SOLR-8393 : Component for Solr resource usage planning
igiguere May 11, 2023
5023255
Merge branch 'apache:main' into SOLR-8393-Component-for-Solr-resource…
igiguere May 11, 2023
d553990
SOLR-8393 : Size component for resource planning
igiguere May 12, 2023
8c64d89
Merge branch 'apache:main' into SOLR-8393-Component-for-Solr-resource…
igiguere Jan 23, 2024
106c1f2
Merge branch 'apache:main' into SOLR-8393-Component-for-Solr-resource…
igiguere Mar 15, 2024
3491a96
Merge branch 'apache:main' into SOLR-8393-Component-for-Solr-resource…
igiguere Apr 2, 2024
c5cbf67
SOLR-8393: Component for Solr resource planning
igiguere Apr 8, 2024
6b9f0ba
Merge branch 'apache:main' into SOLR-8393-Component-for-Solr-resource…
igiguere Apr 8, 2024
4774830
Merge branch 'SOLR-8393-Component-for-Solr-resource-usage-planning' o…
igiguere Apr 8, 2024
16c53d6
Merge branch 'apache:main' into SOLR-8393-Component-for-Solr-resource…
igiguere Apr 10, 2024
92a6ddd
SOLR-8393: Address review comment - SizeComponent
igiguere Apr 10, 2024
38b6167
SOLR-8393: cluster sizing
igiguere Apr 10, 2024
29e8226
Merge branch 'apache:main' into SOLR-8393-Component-for-Solr-resource…
igiguere Apr 14, 2024
0110a0b
SOLR-8393: Solr resources usage planning
igiguere Apr 14, 2024
253c111
SOLR-8393 : Solr resources usage planning
igiguere Apr 14, 2024
6fffece
Merge branch 'SOLR-8393-Component-for-Solr-resource-usage-planning' o…
igiguere Apr 14, 2024
4b8ddd1
Merge branch 'apache:main' into SOLR-8393-Component-for-Solr-resource…
igiguere Jun 7, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.api.endpoint;

import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.parameters.RequestBody;
import jakarta.ws.rs.POST;
import jakarta.ws.rs.Path;
import org.apache.solr.client.api.model.ClusterSizingRequestBody;
// import org.apache.solr.client.api.model.ClusterSizingResponse;
import org.apache.solr.client.api.model.SolrJerseyResponse;

/** JAX-RS V2 API definition for "cluster sizing" (future cluster size estimate) */
@Path("/cluster/size/estimate")
public interface ClusterSizingApi {

@POST
@Operation(
summary = "Get size estimates for resource usage planning based on the current cluster.",
tags = {"cluster"})
// TODO: return ClusterSizingResponse
public SolrJerseyResponse getMetricsEstimate(
@RequestBody(description = "Contains user provided parameters")
final ClusterSizingRequestBody request)
throws Exception;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.api.model;

import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.List;

public class ClusterSizingCollectionDetails {

@JsonProperty public String name;
@JsonProperty public List<Shard> shards;

public class Shard {
@JsonProperty public String name;
@JsonProperty public List<CoreNode> coreNodes;
}

public class CoreNode {
@JsonProperty public String name;

@JsonProperty("details")
public CoreDetails coreDetails;
}

public class CoreDetails {
@JsonProperty public String totalDiskSize;

@JsonProperty public String totalLuceneRam;

@JsonProperty public String totalSolrRam;

@JsonProperty public int estimatedNumDocs;

@JsonProperty public String estimatedDocSize;

@JsonProperty public SolrDetails solrDetails;
}

public class SolrDetails {
@JsonProperty public String filterCache;
@JsonProperty public String queryResultCache;
@JsonProperty public String documentCache;
@JsonProperty public String luceneRam;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.api.model;

import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;

public class ClusterSizingRequestBody {

@Schema(
description =
"Document size used to compute resource usage. If less than 1, the value will be computed using the content of currently indexed documents.")
@JsonProperty("avgDocSize")
public Long avgDocSize;

@Schema(
description =
"Number of documents to use when computing resource usage. If less than 1, actual number of indexed documents will be used. This parameter will be ignored if estimationRatio is specified.")
@JsonProperty("numDocs")
public Long numDocs;

@Schema(
description =
"If specified, will be used as number of deleted documents in the index when computing resource usage, otherwise, current number of deleted documents will be used instead.")
@JsonProperty("deletedDocs")
public Long deletedDocs;

@Schema(
description =
"Size of the filter cache to use for computing resource usage, if not specified, current filter cache size will be used.")
@JsonProperty("filterCacheMax")
public Long filterCacheMax;

@Schema(
description =
"Size of the query result cache to use for computing resource usage, if not specified, current query result cache size will be used.")
@JsonProperty("queryResultCacheMax")
public Long queryResultCacheMax;

@Schema(
description =
"Size of the document cache to use for computing resource usage, if not specified, current document cache size will be used.")
@JsonProperty("documentCacheMax")
public Long documentCacheMax;

@Schema(
description =
"Maximum number of documents to cache per entry in query result cache to use for computing resource usage, if not specified, current maximum will be used.")
@JsonProperty("queryResultMaxDocsCached")
public Long queryResultMaxDocsCached;

@Schema(
description =
"Ratio used for resource usage estimations. If a value greater than 0.0 is specified, the current number of documents will be multiplied by this ratio in order to determine number of documents to be used when computing resource usage.")
@JsonProperty("estimationRatio")
public Double estimationRatio;

@Schema(
description =
"If present, values will be output as 'double', according to the chosen size unit. Default behavior, if not present is a human-readable format. Allowed values are: GB, MB, KB, bytes")
@JsonProperty("sizeUnit")
public String sizeUnit;

public ClusterSizingRequestBody() {}

public ClusterSizingRequestBody(
final Long avgDocSize,
final Long numDocs,
final Long deletedDocs,
final Long filterCacheMax,
final Long queryResultCacheMax,
final Long documentCacheMax,
final Long queryResultMaxDocsCached,
final Double estimationRatio,
final String sizeUnit) {
this.avgDocSize = avgDocSize;
this.numDocs = numDocs;
this.deletedDocs = deletedDocs;
this.filterCacheMax = filterCacheMax;
this.queryResultCacheMax = queryResultCacheMax;
this.documentCacheMax = documentCacheMax;
this.queryResultMaxDocsCached = queryResultMaxDocsCached;
this.estimationRatio = estimationRatio;
this.sizeUnit = sizeUnit;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.api.model;

import com.fasterxml.jackson.annotation.JsonProperty;
import java.util.List;

public class ClusterSizingResponse extends SolrJerseyResponse {

@JsonProperty public ClusterDetails cluster;

public class ClusterDetails {
@JsonProperty public List<Node> nodes;

@JsonProperty public List<ClusterSizingCollectionDetails> collections;
}

public class Node {
@JsonProperty public String name;

@JsonProperty("details")
public List<NodeDetails> nodeDetails;
}

public class NodeDetails {
@JsonProperty public String totalLuceneRam;

@JsonProperty public String totalSolrRam;

@JsonProperty public String totalDiskSize;

@JsonProperty public int estimatedNumDocs;
}
}
14 changes: 14 additions & 0 deletions solr/core/src/java/org/apache/solr/handler/ClusterAPI.java
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,20 @@ public void getClusterStatus(SolrQueryRequest req, SolrQueryResponse rsp) throws
collectionsHandler.handleRequestBody(wrapParams(req, v1Params), rsp);
}

/**
* V2 cluster sizing that just calls the V1 cluster sizing, with dependency on the SizeCompoment.
*
* @param req request
* @param rsp response
* @throws Exception If there is an error.
*/
@EndPoint(method = GET, path = "/cluster/sizing", permission = COLL_READ_PERM)
public void getClusterSizing(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
final Map<String, Object> v1Params =
Map.of(CommonParams.ACTION, CollectionAction.CLUSTERSIZING.toLower());
collectionsHandler.handleRequestBody(wrapParams(req, v1Params), rsp);
}

private CoreContainer getCoreContainer() {
return collectionsHandler.getCoreContainer();
}
Expand Down
Loading
Loading