Skip to content

Commit

Permalink
Fix message when skipping compaction (apache#15460)
Browse files Browse the repository at this point in the history
  • Loading branch information
kfaraz authored and ythorat2 committed Dec 1, 2023
1 parent 5ca9c84 commit 06bbcdb
Showing 1 changed file with 49 additions and 28 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@
import org.apache.druid.timeline.Partitions;
import org.apache.druid.timeline.SegmentTimeline;
import org.apache.druid.timeline.TimelineObjectHolder;
import org.apache.druid.timeline.VersionedIntervalTimeline;
import org.apache.druid.timeline.partition.NumberedPartitionChunk;
import org.apache.druid.timeline.partition.NumberedShardSpec;
import org.apache.druid.timeline.partition.PartitionChunk;
Expand Down Expand Up @@ -71,7 +70,7 @@ public class NewestSegmentFirstIterator implements CompactionSegmentIterator
private final Map<String, CompactionStatistics> compactedSegmentStats = new HashMap<>();
private final Map<String, CompactionStatistics> skippedSegmentStats = new HashMap<>();

private final Map<String, CompactibleTimelineObjectHolderCursor> timelineIterators;
private final Map<String, CompactibleSegmentIterator> timelineIterators;

// This is needed for datasource that has segmentGranularity configured
// If configured segmentGranularity in config is finer than current segmentGranularity, the same set of segments
Expand Down Expand Up @@ -149,10 +148,18 @@ public class NewestSegmentFirstIterator implements CompactionSegmentIterator
originalTimeline = timeline;
timeline = timelineWithConfiguredSegmentGranularity;
}
final List<Interval> searchIntervals =
findInitialSearchInterval(dataSource, timeline, config.getSkipOffsetFromLatest(), configuredSegmentGranularity, skipIntervals.get(dataSource));
final List<Interval> searchIntervals = findInitialSearchInterval(
dataSource,
timeline,
config.getSkipOffsetFromLatest(),
configuredSegmentGranularity,
skipIntervals.get(dataSource)
);
if (!searchIntervals.isEmpty()) {
timelineIterators.put(dataSource, new CompactibleTimelineObjectHolderCursor(timeline, searchIntervals, originalTimeline));
timelineIterators.put(
dataSource,
new CompactibleSegmentIterator(timeline, searchIntervals, originalTimeline)
);
}
}
});
Expand Down Expand Up @@ -218,41 +225,46 @@ private void updateQueue(String dataSourceName, DataSourceCompactionConfig confi
}

/**
* Iterates the given {@link VersionedIntervalTimeline}. Only compactible {@link TimelineObjectHolder}s are returned,
* which means the holder always has at least one {@link DataSegment}.
* Iterates compactible segments in a {@link SegmentTimeline}.
*/
private static class CompactibleTimelineObjectHolderCursor implements Iterator<List<DataSegment>>
private static class CompactibleSegmentIterator implements Iterator<List<DataSegment>>
{
private final List<TimelineObjectHolder<String, DataSegment>> holders;
@Nullable
private final SegmentTimeline originalTimeline;

CompactibleTimelineObjectHolderCursor(
CompactibleSegmentIterator(
SegmentTimeline timeline,
List<Interval> totalIntervalsToSearch,
// originalTimeline can be null if timeline was not modified
@Nullable SegmentTimeline originalTimeline
)
{
this.holders = totalIntervalsToSearch
.stream()
.flatMap(interval -> timeline
this.holders = totalIntervalsToSearch.stream().flatMap(
interval -> timeline
.lookup(interval)
.stream()
.filter(holder -> isCompactibleHolder(interval, holder))
)
.collect(Collectors.toList());
).collect(Collectors.toList());
this.originalTimeline = originalTimeline;
}

private boolean isCompactibleHolder(Interval interval, TimelineObjectHolder<String, DataSegment> holder)
/**
* Checks if the {@link TimelineObjectHolder} satisfies the following:
* <ul>
* <li>It has atleast one segment.</li>
* <li>The interval of the segments is contained in the searchInterval.</li>
* <li>The total bytes across all the segments is positive.</li>
* </ul>
*/
private boolean isCompactibleHolder(Interval searchInterval, TimelineObjectHolder<String, DataSegment> holder)
{
final Iterator<PartitionChunk<DataSegment>> chunks = holder.getObject().iterator();
if (!chunks.hasNext()) {
return false; // There should be at least one chunk for a holder to be compactible.
return false;
}
PartitionChunk<DataSegment> firstChunk = chunks.next();
if (!interval.contains(firstChunk.getObject().getInterval())) {
if (!searchInterval.contains(firstChunk.getObject().getInterval())) {
return false;
}
long partitionBytes = firstChunk.getObject().getSize();
Expand All @@ -268,10 +280,19 @@ public boolean hasNext()
return !holders.isEmpty();
}

/**
* Returns the next list of compactible segments in the datasource timeline.
* The returned list satisfies the following conditions:
* <ul>
* <li>The list is non-null and non-empty.</li>
* <li>The segments are present in the search interval.</li>
* <li>Total bytes of segments in the list is greater than zero.</li>
* </ul>
*/
@Override
public List<DataSegment> next()
{
if (holders.isEmpty()) {
if (!hasNext()) {
throw new NoSuchElementException();
}
TimelineObjectHolder<String, DataSegment> timelineObjectHolder = holders.remove(holders.size() - 1);
Expand Down Expand Up @@ -302,20 +323,20 @@ private SegmentsToCompact findSegmentsToCompact(
final DataSourceCompactionConfig config
)
{
final CompactibleTimelineObjectHolderCursor compactibleTimelineObjectHolderCursor
final CompactibleSegmentIterator compactibleSegmentIterator
= timelineIterators.get(dataSourceName);
if (compactibleTimelineObjectHolderCursor == null) {
log.warn("Skipping dataSource[%s] as there is no timeline for it.", dataSourceName);
if (compactibleSegmentIterator == null) {
log.warn(
"Skipping compaction for datasource[%s] as there is no compactible segment in its timeline.",
dataSourceName
);
return SegmentsToCompact.empty();
}

final long inputSegmentSize = config.getInputSegmentSizeBytes();

while (compactibleTimelineObjectHolderCursor.hasNext()) {
List<DataSegment> segments = compactibleTimelineObjectHolderCursor.next();
if (segments.isEmpty()) {
throw new ISE("No segment is found?");
}
while (compactibleSegmentIterator.hasNext()) {
List<DataSegment> segments = compactibleSegmentIterator.next();

final SegmentsToCompact candidates = SegmentsToCompact.from(segments);
final Interval interval = candidates.getUmbrellaInterval();
Expand Down Expand Up @@ -353,7 +374,7 @@ private SegmentsToCompact findSegmentsToCompact(
}
}

log.debug("All segments look good! Nothing to compact");
log.debug("No more segments to compact for datasource[%s].", dataSourceName);
return SegmentsToCompact.empty();
}

Expand All @@ -377,7 +398,7 @@ private void addSegmentStatsTo(
*/
private List<Interval> findInitialSearchInterval(
String dataSourceName,
VersionedIntervalTimeline<String, DataSegment> timeline,
SegmentTimeline timeline,
Period skipOffset,
Granularity configuredSegmentGranularity,
@Nullable List<Interval> skipIntervals
Expand Down

0 comments on commit 06bbcdb

Please sign in to comment.