Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,12 @@ private CommitBatchResponse commitFileForValidRequest(
);
coordinates.put(lastOffset, new BatchInfoInternal(batchInfo, fileInfo));

// Populate only when unset: appends add newer batches at the head, so a populated log's oldest
// batch is unchanged. Covers empty->non-empty and lazily backfills a still-null log.
if (logInfo.earliestBatchTimestamp == null) {
logInfo.earliestBatchTimestamp = earliestBatchTimestamp(coordinates);
}

fileInfo.addBatch(batchInfo);

return CommitBatchResponse.success(firstOffset, now, logInfo.logStartOffset, fileInfo.objectKey, request);
Expand Down Expand Up @@ -329,7 +335,8 @@ private DeleteRecordsResponse deleteRecordsForPartition(final DeleteRecordsReque
if (convertedOffset < 0 || convertedOffset > logInfo.highWatermark) {
return DeleteRecordsResponse.offsetOutOfRange();
}
if (convertedOffset > logInfo.logStartOffset) {
final boolean advanced = convertedOffset > logInfo.logStartOffset;
if (advanced) {
logInfo.logStartOffset = convertedOffset;
}

Expand All @@ -340,6 +347,10 @@ private DeleteRecordsResponse deleteRecordsForPartition(final DeleteRecordsReque
logInfo.byteSize -= batchInfoInternal.batchInfo().metadata().byteSize();
assert logInfo.byteSize >= 0;
}
// Recompute only on advance (null when the log emptied); a no-op delete leaves it untouched.
if (advanced) {
logInfo.earliestBatchTimestamp = earliestBatchTimestamp(coordinates);
}
return (DeleteRecordsResponse.success(logInfo.logStartOffset));
}

Expand Down Expand Up @@ -387,8 +398,21 @@ public synchronized List<EnforceRetentionResponse> enforceRetention(final List<E
int batchesDeleted = 0;
long bytesDeleted = 0;

// Short-circuit (mirrors enforce_retention_v2): if the oldest retained batch survives every
// enabled policy, nothing is deletable, so skip the O(depth) selection scans below. This is a
// pure optimization - both scans would select nothing in this case (the size scan starts only
// when byteSize exceeds the limit; the time takeWhile stops at the first non-breaching batch),
// so we still fall through to the unchanged tail that reports log_start_offset. A null
// earliestBatchTimestamp means "unknown", so we cannot prove time-safety and run the scans.
final boolean sizeSafe = request.retentionBytes() < 0 || logInfo.byteSize <= request.retentionBytes();
final boolean timeSafe = request.retentionMs() < 0
|| (logInfo.earliestBatchTimestamp != null
&& logInfo.earliestBatchTimestamp >= now.toEpochMilli() - request.retentionMs());
final boolean nothingDeletable = sizeSafe && timeSafe;

// Enforce the size retention.
if (request.retentionBytes() >= 0
if (!nothingDeletable
&& request.retentionBytes() >= 0
// Does it even make sense to start iterating?
&& logInfo.byteSize > request.retentionBytes()) {
long accumulatedSize = 0;
Expand All @@ -402,7 +426,7 @@ public synchronized List<EnforceRetentionResponse> enforceRetention(final List<E
}

// Enforce the time retention.
if (request.retentionMs() >= 0) {
if (!nothingDeletable && request.retentionMs() >= 0) {
final long lastRetainedTimestamp = now.toEpochMilli() - request.retentionMs();
// Select batches for deletion only while we see them breaching the retention threshold, but no further.
coordinates.values().stream()
Expand All @@ -427,13 +451,23 @@ public synchronized List<EnforceRetentionResponse> enforceRetention(final List<E
}

logInfo.byteSize -= bytesDeleted;
if (coordinates.isEmpty()) {
logInfo.logStartOffset = logInfo.highWatermark;
if (logInfo.byteSize != 0) {
throw new RuntimeException(String.format("Log size expected to be 0, but it's %d", logInfo.byteSize));
// Only a deletion can change log_start or the oldest batch. When nothing was deleted, leave both
// untouched: recomputing log_start from the oldest batch's base offset would move it BACKWARD if a
// prior deleteRecords advanced it into the middle of the still-retained oldest batch, diverging
// from delete_records_v1 and wrongly serving reads below the log start.
if (batchesDeleted > 0) {
if (coordinates.isEmpty()) {
logInfo.logStartOffset = logInfo.highWatermark;
if (logInfo.byteSize != 0) {
throw new RuntimeException(String.format("Log size expected to be 0, but it's %d", logInfo.byteSize));
}
} else {
// Forward-only for the same reason.
logInfo.logStartOffset = Math.max(
logInfo.logStartOffset,
coordinates.firstEntry().getValue().batchInfo().metadata().baseOffset());
}
} else {
logInfo.logStartOffset = coordinates.firstEntry().getValue().batchInfo().metadata().baseOffset();
logInfo.earliestBatchTimestamp = earliestBatchTimestamp(coordinates);
}
responses.add(EnforceRetentionResponse.success(batchesDeleted, bytesDeleted, logInfo.logStartOffset));
}
Expand Down Expand Up @@ -643,6 +677,12 @@ private PruneDisklessLogsResponse pruneDisklessLogsForPartition(final PruneDiskl
}
logInfo.logStartOffset = newLogStart;

// Cross-tier prune removes the oldest batches, so recompute when it deleted something
// (null when the log emptied); no removal leaves it untouched.
if (!keysToRemove.isEmpty()) {
logInfo.earliestBatchTimestamp = earliestBatchTimestamp(coordinates);
}

return new PruneDisklessLogsResponse(requestTip, newLogStart, PruneDisklessLogsError.NONE);
}

Expand All @@ -658,13 +698,27 @@ private TopicIdPartition findTopicIdPartition(final Uuid topicId, final int part
.orElse(null);
}

// Effective timestamp of the oldest retained batch (smallest last-offset key == batch at logStartOffset),
// or null when the log is empty. Matches batch_timestamp() / logs.earliest_batch_timestamp semantics.
private static Long earliestBatchTimestamp(final TreeMap<Long, BatchInfoInternal> coordinates) {
if (coordinates.isEmpty()) {
return null;
}
return coordinates.firstEntry().getValue().batchInfo().metadata().timestamp();
}

private static class LogInfo {
long logStartOffset = 0;
long highWatermark = 0;
long byteSize = 0;
long disklessStartOffset = 0;
// -1 means "no remote tier tracked yet"; mirrors the NULL sentinel of logs.remote_log_start_offset.
long remoteLogStartOffset = -1;
// Effective timestamp of the oldest retained batch (the one at logStartOffset), mirroring
// logs.earliest_batch_timestamp. null means "unknown, must scan" (same as the NULL sentinel in SQL).
// Maintained purely for parity with the Postgres control plane so the enforceRetention short-circuit
// behaves identically across backends.
Long earliestBatchTimestamp = null;
}

private static class FileInfo {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
*/
package io.aiven.inkless.control_plane.postgres;

import org.apache.kafka.common.TopicIdPartition;
import org.apache.kafka.common.utils.Time;

import org.jooq.Configuration;
Expand All @@ -27,6 +26,7 @@
import org.jooq.generated.udt.records.EnforceRetentionResponseV1Record;

import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.function.Consumer;
Expand Down Expand Up @@ -66,16 +66,24 @@ public List<EnforceRetentionResponse> call() throws Exception {
}

private List<EnforceRetentionResponse> runOnce() {
return jooqCtx.transactionResult((final Configuration conf) -> {
final Instant now = TimeUtils.now(time);
final Instant now = TimeUtils.now(time);
final List<EnforceRetentionResponse> responses = new ArrayList<>(requests.size());
for (final EnforceRetentionRequest request : requests) {
responses.add(enforceOne(now, request));
}
return responses;
}

final EnforceRetentionRequestV1Record[] jooqRequests = requests.stream().map(r ->
private EnforceRetentionResponse enforceOne(final Instant now, final EnforceRetentionRequest request) {
return jooqCtx.transactionResult((final Configuration conf) -> {
final EnforceRetentionRequestV1Record[] jooqRequests = {
new EnforceRetentionRequestV1Record(
r.topicId(),
r.partition(),
r.retentionBytes(),
r.retentionMs()
)).toArray(EnforceRetentionRequestV1Record[]::new);
request.topicId(),
request.partition(),
request.retentionBytes(),
request.retentionMs()
)
};

try {
final List<EnforceRetentionResponseV1Record> functionResult = conf.dsl().select(
Expand All @@ -88,31 +96,30 @@ private List<EnforceRetentionResponse> runOnce() {
).from(ENFORCE_RETENTION_V2.call(
now, jooqRequests, maxBatchesPerRequest
)).fetchInto(EnforceRetentionResponseV1Record.class);
return FunctionResultProcessor.processWithMappingOrder(
requests,
functionResult,
// We don't care about the topic name for the key.
r -> new TopicIdPartition(r.topicId(), r.partition(), null),
r -> new TopicIdPartition(r.getTopicId(), r.getPartition(), null),
this::responseMapper
);

if (functionResult.size() != 1) {
throw new RuntimeException("Expected 1 response for " + request + ", got " + functionResult.size());
}
final EnforceRetentionResponseV1Record record = functionResult.get(0);
if (!record.getTopicId().equals(request.topicId()) || record.getPartition() != request.partition()) {
throw new RuntimeException("enforce_retention_v2 returned a response for "
+ record.getTopicId() + "-" + record.getPartition()
+ ", expected " + request.topicId() + "-" + request.partition());
}
return responseMapper(record);
} catch (RuntimeException e) {
throw new ControlPlaneException("Error enforcing retention", e);
}
});
}

private EnforceRetentionResponse responseMapper(final EnforceRetentionRequest request,
final EnforceRetentionResponseV1Record record) {
private EnforceRetentionResponse responseMapper(final EnforceRetentionResponseV1Record record) {
if (record.getError() == null) {
return EnforceRetentionResponse.success(record.getBatchesDeleted(), record.getBytesDeleted(), record.getLogStartOffset());
} else {
return switch (record.getError()) {
case unknown_topic_or_partition ->
EnforceRetentionResponse.unknownTopicOrPartition();
default ->
throw new RuntimeException(String.format("Unknown error '%s' returned for %s-%d",
record.getError(), record.getTopicId(), record.getPartition()));
};
}
}
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion storage/inkless/src/main/jooq/org/jooq/generated/Keys.java

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion storage/inkless/src/main/jooq/org/jooq/generated/UDTs.java

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading