Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -807,7 +807,7 @@ abstract class AbstractFetcherThread(name: String,
}
}

private def delayPartitions(partitions: Iterable[TopicPartition], delay: Long): Unit = {
protected def delayPartitions(partitions: Iterable[TopicPartition], delay: Long): Unit = {
partitionMapLock.lockInterruptibly()
try {
for (partition <- partitions) {
Expand Down
35 changes: 30 additions & 5 deletions core/src/main/scala/kafka/server/ReplicaFetcherThread.scala
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ class ReplicaFetcherThread(name: String,
// and should be evicted from this fetcher.
private[server] val partitionsToEvictAfterDisklessSwitch = mutable.Buffer[TopicPartition]()

// At the seal but not yet in metadata ISR, so we cannot evict. Visible for testing.
private[server] val partitionsAwaitingIsrRecovery = mutable.Buffer[TopicPartition]()

override protected def latestEpoch(topicPartition: TopicPartition): Optional[Integer] = {
replicaMgr.localLogOrException(topicPartition).latestEpoch
}
Expand Down Expand Up @@ -100,6 +103,7 @@ class ReplicaFetcherThread(name: String,
super.doWork()
completeDelayedFetchRequests()
evictFullySwitchedDisklessPartitions()
backOffPartitionsAwaitingIsrRecovery()
}

/**
Expand Down Expand Up @@ -168,14 +172,25 @@ class ReplicaFetcherThread(name: String,
if (shouldRecordReplicationBytesIn)
brokerTopicStats.updateReplicationBytesIn(records.sizeInBytes)

// Stop fetching after the switch from classic to diskless is completed: once the controller
// has committed a classicToDisklessStartOffset for this partition AND our local LEO has reached it,
// the follower is fully caught up to the leader's frozen classic log and must not keep fetching.
val classicToDisklessStartOffset = replicaMgr.inklessMetadataView().getClassicToDisklessStartOffset(topicPartition)
// Stop fetching once the switch is complete: seal is committed, local LEO has reached it,
// and this replica is in ISR. A consolidating partition evicts without waiting for ISR so
// it can hand off to the consolidation fetcher.
val inklessMetadataView = replicaMgr.inklessMetadataView()
val classicToDisklessStartOffset = inklessMetadataView.getClassicToDisklessStartOffset(topicPartition)
def isConsolidatingPartition: Boolean =
brokerConfig.disklessRemoteStorageConsolidationEnabled &&
inklessMetadataView.isConsolidatingDisklessTopic(topicPartition.topic)
if (shouldEvictFullySwitchedDisklessPartitions &&
classicToDisklessStartOffset >= 0 &&
log.logEndOffset >= classicToDisklessStartOffset) {
partitionsToEvictAfterDisklessSwitch += topicPartition
if (isConsolidatingPartition || inklessMetadataView.isReplicaInIsr(topicPartition, brokerConfig.brokerId)) {
partitionsToEvictAfterDisklessSwitch += topicPartition
} else {
// The leader answers this fetch from immediateFetchResponses and does not park it
// in the fetch purgatory, so maxWaitMs is ignored. Delay here or we re-fetch at
// network rate until the ISR expansion lands.
partitionsAwaitingIsrRecovery += topicPartition
}
}

logAppendInfo
Expand All @@ -188,6 +203,16 @@ class ReplicaFetcherThread(name: String,
}
}

// Visible for testing. Must run from doWork, not processPartitionData: processFetchRequest
// overwrites fetch state right after processPartitionData and would drop an inline delay.
private[server] def backOffPartitionsAwaitingIsrRecovery(): Unit = {
if (partitionsAwaitingIsrRecovery.nonEmpty) {
val toDelay = partitionsAwaitingIsrRecovery.toSet
partitionsAwaitingIsrRecovery.clear()
delayPartitions(toDelay, brokerConfig.replicaFetchBackoffMs.toLong)
}
}

private def evictFullySwitchedDisklessPartitions(): Unit = {
if (partitionsToEvictAfterDisklessSwitch.nonEmpty) {
val toEvict = partitionsToEvictAfterDisklessSwitch.toSet
Expand Down
74 changes: 60 additions & 14 deletions core/src/main/scala/kafka/server/ReplicaManager.scala
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ import org.apache.kafka.common.message.ListOffsetsResponseData.{ListOffsetsParti
import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.{OffsetForLeaderPartition, OffsetForLeaderTopic}
import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.{EpochEndOffset, OffsetForLeaderTopicResult}
import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse
import org.apache.kafka.common.message.{DescribeLogDirsResponseData, DescribeProducersResponseData}
import org.apache.kafka.common.message.{DescribeLogDirsResponseData, DescribeProducersResponseData, FetchResponseData}
import org.apache.kafka.common.metrics.Metrics
import org.apache.kafka.common.network.ListenerName
import org.apache.kafka.common.protocol.Errors
Expand Down Expand Up @@ -91,6 +91,7 @@ import java.util.stream.Collectors
import scala.collection.{Map, Seq, Set, immutable, mutable}
import scala.jdk.CollectionConverters._
import scala.jdk.OptionConverters.RichOptional
import scala.util.control.NonFatal

/*
* Result metadata of a log append operation on the log
Expand Down Expand Up @@ -2508,20 +2509,59 @@ class ReplicaManager(val config: KafkaConfig,
if (!partitionLookupFailed) {
val disklessSwitchCompleted = !shouldReadFromUnifiedLog && classicToDisklessStartOffset >= 0
if (params.isFromFollower && disklessSwitchCompleted) {
var fetchError = Errors.NONE
var divergingEpoch = Optional.empty[FetchResponseData.EpochEndOffset]
// A recovered follower for a switched partition may already be caught up to the
// seal offset but still be outside ISR. Record the seal-offset fetch so the normal
// ISR expansion path can observe that the follower is caught up without reading
// diskless data into the local log.
if (fetchPartitionData.fetchOffset >= classicToDisklessStartOffset) {
getPartitionOrError(tp.topicPartition) match {
case Right(partition) =>
try {
// Use the classic follower-read validation without returning any records.
val fetchAtSeal = new PartitionData(
fetchPartitionData.topicId,
classicToDisklessStartOffset,
fetchPartitionData.logStartOffset,
0,
fetchPartitionData.currentLeaderEpoch,
fetchPartitionData.lastFetchedEpoch
)
val readInfo = partition.fetchRecords(
fetchParams = params,
fetchPartitionData = fetchAtSeal,
fetchTimeMs = time.milliseconds,
maxBytes = 0,
minOneMessage = false,
updateFetchState = true
)
divergingEpoch = readInfo.divergingEpoch
} catch {
case NonFatal(e) =>
fetchError = Errors.forException(e)
Comment thread
viktorsomogyi marked this conversation as resolved.
Resolved
if (fetchError == Errors.UNKNOWN_SERVER_ERROR) {
error(s"Error validating at-seal fetch from " +
s"${FetchRequest.describeReplicaId(params.replicaId)} on partition $tp " +
s"at seal $classicToDisklessStartOffset", e)
}
}
case Left(error) => fetchError = error
}
}
// The partition has fully switched to diskless and the follower is asking for an offset at or beyond it.
// Followers must never replicate diskless records into their local log. Return
// an empty response with HW clamped to the seal offset so the fetcher loop sees the
// partition as caught up and goes idle, rather than treating it as out of range.
// Deliberately pass logStartOffset=0 (a no-op for the follower since
// maybeIncrementLogStartOffset only ever advances) so the follower keeps its classic
// local data intact and remains able to serve consumer reads from the local log.
// Followers must never replicate diskless records into their local log.
// Empty records and HW at the seal offset make the follower treat the local log as caught up.
// ReplicaFetcherThread evicts once this replica is in ISR, or immediately if consolidating.
// logStartOffset=0 is a no-op for the follower (maybeIncrementLogStartOffset only ever advances),
// so classic local data stays in place and can still serve consumer reads.
immediateFetchResponses += tp ->
new FetchPartitionData(
Errors.NONE,
classicToDisklessStartOffset,
0L,
fetchError,
if (fetchError == Errors.NONE) classicToDisklessStartOffset else UnifiedLog.UNKNOWN_OFFSET,
if (fetchError == Errors.NONE) 0L else UnifiedLog.UNKNOWN_OFFSET,
MemoryRecords.EMPTY,
Optional.empty(),
divergingEpoch,
OptionalLong.empty(),
Optional.empty(),
OptionalInt.empty(),
Expand Down Expand Up @@ -4102,11 +4142,17 @@ class ReplicaManager(val config: KafkaConfig,
val isNewLeaderEpoch = partition.makeFollower(info.partition, isNew, offsetCheckpoints, Some(info.topicId), partitionAssignedDirectoryId)
partition.seal()
changedPartitions.add(partition)
if (seal >= 0 && partition.localLogOrException.highWatermark < seal) {
val isOutOfIsr = !info.partition.isr.contains(config.brokerId)
// Skip during controlled shutdown: the leader will not expand ISR for a shutting-down
// broker (isReplicaIsrEligible), and this replica is about to stop serving.
if (seal >= 0 && !isInControlledShutdown &&
(partition.localLogOrException.highWatermark < seal || isOutOfIsr)) {
// Schedule a catch-up fetch when the local HW is below the seal -- either
// because we restarted with a stale HW (unclean shutdown) or because we
// were just added as a replica and have an empty local log. The
// ReplicaFetcher self-evicts once the follower has read past the seal.
// were just added as a replica and have an empty local log.
// Also schedule one when this replica is caught up but out of ISR, so the leader
// observes its fetch state and can expand ISR.
// Eviction waits for ISR membership, not just for the seal -- see ReplicaFetcherThread.processPartitionData.
partitionsToStartFetching.put(tp, partition)
} else if (seal == PartitionRegistration.CLASSIC_TO_DISKLESS_SWITCH_PENDING && isNewLeaderEpoch) {
// Switch is in flight: the leader has already sealed its log and
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,12 @@ class InklessMetadataView(val metadataCache: KRaftMetadataCache, val defaultConf
.getOrElse(PartitionRegistration.NO_CLASSIC_TO_DISKLESS_START_OFFSET)
}

def isReplicaInIsr(topicPartition: TopicPartition, replicaId: Int): Boolean = {
Option(metadataCache.currentImage().topics().getTopic(topicPartition.topic()))
.flatMap(topicImage => Option(topicImage.partitions().get(topicPartition.partition())))
.exists(_.isr.contains(replicaId))
}

/**
* The diskless leader epoch (E_d) captured at the classic-to-diskless switch, or
* [[PartitionRegistration.NO_DISKLESS_LEADER_EPOCH]] when the partition never switched (born-diskless
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,21 @@ class InklessMetadataViewTest {
assertEquals(PartitionRegistration.NO_CLASSIC_TO_DISKLESS_START_OFFSET, metadataView.getClassicToDisklessStartOffset(tp))
}

@Test
def testIsReplicaInIsrUsesImageState(): Unit = {
val tp = new TopicPartition("switched", 0)
stubImageTopic(tp.topic(), util.Map.of(Integer.valueOf(0), partitionRegistration()))
assertTrue(metadataView.isReplicaInIsr(tp, 1))
assertFalse(metadataView.isReplicaInIsr(tp, 2))
}

@Test
def testIsReplicaInIsrReturnsFalseWhenTopicMissing(): Unit = {
val tp = new TopicPartition("missing", 0)
stubImageWithoutTopic(tp.topic())
assertFalse(metadataView.isReplicaInIsr(tp, 1))
}

@Nested
class TopicConfigCacheTest {
@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -759,6 +759,74 @@ class ReplicaFetcherThreadTest {
expectEviction = true)
}

@Test
def shouldNotEvictPartitionAtSealUntilMetadataIsrContainsReplica(): Unit = {
verifyDisklessSwitchEviction(
classicToDisklessStartOffset = 100L,
logEndOffsetAfterAppend = 100L,
expectEviction = false,
replicaInIsr = false)
}

@Test
def shouldDelayPartitionAtSealWhileMetadataIsrDoesNotContainReplica(): Unit = {
val props = TestUtils.createBrokerConfig(1)
val config = KafkaConfig.fromProps(props)

val mockBlockingSend: BlockingSend = mock(classOf[BlockingSend])
when(mockBlockingSend.brokerEndPoint()).thenReturn(brokerEndPoint)

val log: UnifiedLog = mock(classOf[UnifiedLog])
when(log.logEndOffset).thenReturn(100L)
when(log.latestEpoch).thenReturn(Optional.of(Integer.valueOf(1)))
when(log.maybeUpdateHighWatermark(anyLong())).thenReturn(Optional.empty)

val partition: Partition = mock(classOf[Partition])
when(partition.localLogOrException).thenReturn(log)
when(partition.appendRecordsToFollowerOrFutureReplica(any[MemoryRecords], any[Boolean], any[Int]))
.thenReturn(Some(mock(classOf[LogAppendInfo])))

val inklessMetadataView: InklessMetadataView = mock(classOf[InklessMetadataView])
when(inklessMetadataView.getClassicToDisklessStartOffset(t1p0)).thenReturn(100L)
when(inklessMetadataView.isReplicaInIsr(t1p0, config.brokerId)).thenReturn(false)

val replicaFetcherManager: ReplicaFetcherManager = mock(classOf[ReplicaFetcherManager])
val replicaManager: ReplicaManager = mock(classOf[ReplicaManager])
when(replicaManager.getPartitionOrException(any[TopicPartition])).thenReturn(partition)
when(replicaManager.localLogOrException(t1p0)).thenReturn(log)
when(replicaManager.brokerTopicStats).thenReturn(new BrokerTopicStats)
when(replicaManager.inklessMetadataView()).thenReturn(inklessMetadataView)
when(replicaManager.replicaFetcherManager).thenReturn(replicaFetcherManager)

val thread = createReplicaFetcherThread(
name = "replica-fetcher",
fetcherId = 0,
brokerConfig = config,
failedPartitions = failedPartitions,
replicaMgr = replicaManager,
quota = mock(classOf[ReplicaQuota]),
leaderEndpointBlockingSend = mockBlockingSend)

thread.addPartitions(Map(t1p0 -> initialFetchState(Some(topicId1), 100L)))
val partitionData = new FetchResponseData.PartitionData()
.setPartitionIndex(t1p0.partition)
.setRecords(MemoryRecords.withRecords(Compression.NONE,
new SimpleRecord(1000, "foo".getBytes(StandardCharsets.UTF_8))))
thread.processPartitionData(t1p0, 100L, Int.MaxValue, partitionData)

assertFalse(thread.fetchState(t1p0).get.isDelayed,
"Delay must be applied from doWork, not inline in processPartitionData")
verify(replicaFetcherManager, times(0)).removeFetcherForPartitions(any())

thread.backOffPartitionsAwaitingIsrRecovery()

val fetchState = thread.fetchState(t1p0).get
assertTrue(fetchState.isDelayed,
s"Partition must be delayed after the at-seal fetch, got $fetchState")
assertEquals(config.replicaFetchBackoffMs.toLong, fetchState.delay.orElse(0L))
assertEquals(mutable.Buffer.empty, thread.partitionsAwaitingIsrRecovery)
}

@Test
def shouldNotEvictPartitionWhenLogEndOffsetBelowClassicToDisklessSealOffset(): Unit = {
verifyDisklessSwitchEviction(
Expand Down Expand Up @@ -786,7 +854,8 @@ class ReplicaFetcherThreadTest {
private def verifyDisklessSwitchEviction(
classicToDisklessStartOffset: Long,
logEndOffsetAfterAppend: Long,
expectEviction: Boolean
expectEviction: Boolean,
replicaInIsr: Boolean = true
): Unit = {
val props = TestUtils.createBrokerConfig(1)
val config = KafkaConfig.fromProps(props)
Expand All @@ -802,11 +871,13 @@ class ReplicaFetcherThreadTest {

val partition: Partition = mock(classOf[Partition])
when(partition.localLogOrException).thenReturn(log)
when(partition.inSyncReplicaIds).thenReturn(Set.empty)
when(partition.appendRecordsToFollowerOrFutureReplica(any[MemoryRecords], any[Boolean], any[Int]))
.thenReturn(Some(mock(classOf[LogAppendInfo])))

val inklessMetadataView: InklessMetadataView = mock(classOf[InklessMetadataView])
when(inklessMetadataView.getClassicToDisklessStartOffset(t1p0)).thenReturn(classicToDisklessStartOffset)
when(inklessMetadataView.isReplicaInIsr(t1p0, config.brokerId)).thenReturn(replicaInIsr)

val replicaFetcherManager: ReplicaFetcherManager = mock(classOf[ReplicaFetcherManager])

Expand Down Expand Up @@ -849,6 +920,7 @@ class ReplicaFetcherThreadTest {
verify(replicaManager, times(0)).startConsolidationFetchersForCaughtUpClassicPartitions(any())
}
assertEquals(mutable.Buffer.empty, thread.partitionsToEvictAfterDisklessSwitch)
assertEquals(mutable.Buffer.empty, thread.partitionsAwaitingIsrRecovery)
}

private def newOffsetForLeaderPartitionResult(
Expand Down
Loading
Loading