-
Notifications
You must be signed in to change notification settings - Fork 25k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[6.8] Fix delete_expired_data/nightly maintenance when many model snapshots need deleting #57174
Merged
Merged
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -88,7 +88,14 @@ protected void removeDataBefore(Job job, long cutoffEpochMs, ActionListener<Bool | |
.mustNot(activeSnapshotFilter) | ||
.mustNot(retainFilter); | ||
|
||
searchRequest.source(new SearchSourceBuilder().query(query).size(MODEL_SNAPSHOT_SEARCH_SIZE).sort(ElasticsearchMappings.ES_DOC)); | ||
SearchSourceBuilder source = new SearchSourceBuilder(); | ||
source.query(query); | ||
source.size(MODEL_SNAPSHOT_SEARCH_SIZE); | ||
source.sort(ElasticsearchMappings.ES_DOC); | ||
source.fetchSource(false); | ||
source.docValueField(Job.ID.getPreferredName(), null); | ||
source.docValueField(ModelSnapshotField.SNAPSHOT_ID.getPreferredName(), null); | ||
searchRequest.source(source); | ||
|
||
getClient().execute(SearchAction.INSTANCE, searchRequest, new ThreadedActionListener<>(LOGGER, threadPool, | ||
MachineLearning.UTILITY_THREAD_POOL_NAME, expiredSnapshotsListener(job.getId(), listener), false)); | ||
|
@@ -99,11 +106,18 @@ private ActionListener<SearchResponse> expiredSnapshotsListener(String jobId, Ac | |
@Override | ||
public void onResponse(SearchResponse searchResponse) { | ||
try { | ||
List<ModelSnapshot> modelSnapshots = new ArrayList<>(); | ||
List<JobSnapshotId> snapshotIds = new ArrayList<>(); | ||
for (SearchHit hit : searchResponse.getHits()) { | ||
modelSnapshots.add(ModelSnapshot.fromJson(hit.getSourceRef())); | ||
JobSnapshotId idPair = new JobSnapshotId( | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The logic here is different to the later branches as it doesn't have the new model snapshot retention options added in #56125 |
||
stringFieldValueOrNull(hit, Job.ID.getPreferredName()), | ||
stringFieldValueOrNull(hit, ModelSnapshotField.SNAPSHOT_ID.getPreferredName())); | ||
|
||
if (idPair.hasNullValue() == false) { | ||
snapshotIds.add(idPair); | ||
} | ||
} | ||
deleteModelSnapshots(new VolatileCursorIterator<>(modelSnapshots), listener); | ||
|
||
deleteModelSnapshots(new VolatileCursorIterator<>(snapshotIds), listener); | ||
} catch (Exception e) { | ||
onFailure(e); | ||
} | ||
|
@@ -116,14 +130,14 @@ public void onFailure(Exception e) { | |
}; | ||
} | ||
|
||
private void deleteModelSnapshots(Iterator<ModelSnapshot> modelSnapshotIterator, ActionListener<Boolean> listener) { | ||
private void deleteModelSnapshots(Iterator<JobSnapshotId> modelSnapshotIterator, ActionListener<Boolean> listener) { | ||
if (modelSnapshotIterator.hasNext() == false) { | ||
listener.onResponse(true); | ||
return; | ||
} | ||
ModelSnapshot modelSnapshot = modelSnapshotIterator.next(); | ||
DeleteModelSnapshotAction.Request deleteSnapshotRequest = new DeleteModelSnapshotAction.Request( | ||
modelSnapshot.getJobId(), modelSnapshot.getSnapshotId()); | ||
JobSnapshotId idPair = modelSnapshotIterator.next(); | ||
DeleteModelSnapshotAction.Request deleteSnapshotRequest = | ||
new DeleteModelSnapshotAction.Request(idPair.jobId, idPair.snapshotId); | ||
getClient().execute(DeleteModelSnapshotAction.INSTANCE, deleteSnapshotRequest, new ActionListener<AcknowledgedResponse>() { | ||
@Override | ||
public void onResponse(AcknowledgedResponse response) { | ||
|
@@ -136,9 +150,23 @@ public void onResponse(AcknowledgedResponse response) { | |
|
||
@Override | ||
public void onFailure(Exception e) { | ||
listener.onFailure(new ElasticsearchException("[" + modelSnapshot.getJobId() + "] Failed to delete snapshot [" | ||
+ modelSnapshot.getSnapshotId() + "]", e)); | ||
listener.onFailure(new ElasticsearchException("[" + idPair.jobId + "] Failed to delete snapshot [" | ||
+ idPair.snapshotId + "]", e)); | ||
} | ||
}); | ||
} | ||
|
||
static class JobSnapshotId { | ||
private final String jobId; | ||
private final String snapshotId; | ||
|
||
JobSnapshotId(String jobId, String snapshotId) { | ||
this.jobId = jobId; | ||
this.snapshotId = snapshotId; | ||
} | ||
|
||
boolean hasNullValue() { | ||
return jobId == null || snapshotId == null; | ||
} | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
In 6.8 the doc value could be a
Long
rather than aString
. It's whyTimeField
has this case:elasticsearch/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/extractor/TimeField.java
Line 49 in cde2026
What this means in practice is that a user running 6.8 who first used ML in 5.x will end up seeing the warning on lines 139-140 repeatedly and won't get any cleanup.
It's still OK to use
stringFieldValueOrNull()
to extract fields mapped askeyword
ortext
from hits, but for fields mapped asdate
in 6.8 the code needs to handle bothLong
andString
.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Good point thanks.
I'm curious though why does this code throw if the object is a Long? This is from the 6.8 branch
elasticsearch/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java
Line 118 in 62f8da2
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The tricky part is the condition checks the value is not a
long
. Thus, the logic there is that prior to 6.0, we expect along
. If it's not, then something's gone wrong. Otherwise, we fall through the lastreturn
of the method. Pretty confusing, I know.