Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 14 additions & 2 deletions core/src/main/scala/kafka/server/DelayedFetch.scala
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,14 @@ class DelayedFetch(
val classicParams = replicaManager.fetchParamsWithNewMaxBytes(params, classicPercentage)
val results = replicaManager.readFromLog(classicParams, classicFetchInfos, quota, readFromPurgatory = true)
val resultMap = new util.HashMap[TopicIdPartition, LogReadResult]()
results.foreach { case (tp, r) => resultMap.put(tp, r) }
var consolidationLocalBytes = 0L
results.foreach { case (tp, r) =>
resultMap.put(tp, r)
if (consolidatingSupplements.contains(tp)) {
consolidationLocalBytes += r.info.records.sizeInBytes
}
}
if (consolidationLocalBytes > 0) replicaManager.recordConsolidationLocalBytes(consolidationLocalBytes)
(results, resultMap)
} else (Seq.empty, new util.HashMap[TopicIdPartition, LogReadResult]())

Expand All @@ -262,10 +269,15 @@ class DelayedFetch(
val emptySupplementMap: Map[TopicIdPartition, FetchPartitionData] = Map.empty
val supplementFuture: CompletableFuture[Map[TopicIdPartition, FetchPartitionData]] =
if (supplementFetchInfos.nonEmpty) {
replicaManager.recordConsolidationSupplementIssued()
val supplementParams = replicaManager.fetchParamsWithNewMaxBytes(
params, supplementFetchInfos.size.toFloat / totalRequestsSize)
replicaManager.fetchDisklessMessages(supplementParams, supplementFetchInfos)
.thenApply[Map[TopicIdPartition, FetchPartitionData]](_.toMap)
.thenApply[Map[TopicIdPartition, FetchPartitionData]] { data =>
val dataMap = data.toMap
replicaManager.recordConsolidationSupplementBytes(replicaManager.successfulSupplementBytes(dataMap))
dataMap
}
.exceptionally((e: Throwable) => {
logger.warn("Failed to fetch diskless supplement for consolidating partitions in delayed fetch, returning local data only", e)
emptySupplementMap
Expand Down
44 changes: 40 additions & 4 deletions core/src/main/scala/kafka/server/ReplicaManager.scala
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ import kafka.cluster.Partition
import kafka.log.LogManager
import kafka.server.HostedPartition.Online
import kafka.server.QuotaFactory.QuotaManagers
import kafka.server.ReplicaManager.{AtMinIsrPartitionCountMetricName, ConsolidationFetchBytesInPerSecMetricName, FailedIsrUpdatesPerSecMetricName, IsrExpandsPerSecMetricName, IsrShrinksPerSecMetricName, LeaderCountMetricName, OfflineReplicaCountMetricName, PartitionCountMetricName, PartitionsWithLateTransactionsCountMetricName, ProducerIdCountMetricName, ReassigningPartitionsMetricName, SealedPartitionsCountMetricName, UnderMinIsrPartitionCountMetricName, UnderReplicatedPartitionsMetricName, createLogReadResult, isListOffsetsTimestampUnsupported}
import kafka.server.ReplicaManager.{AtMinIsrPartitionCountMetricName, ConsolidationFetchBytesInPerSecMetricName, ConsolidationLocalBytesPerSecMetricName, ConsolidationSupplementBytesPerSecMetricName, ConsolidationSupplementRateMetricName, FailedIsrUpdatesPerSecMetricName, IsrExpandsPerSecMetricName, IsrShrinksPerSecMetricName, LeaderCountMetricName, OfflineReplicaCountMetricName, PartitionCountMetricName, PartitionsWithLateTransactionsCountMetricName, ProducerIdCountMetricName, ReassigningPartitionsMetricName, SealedPartitionsCountMetricName, UnderMinIsrPartitionCountMetricName, UnderReplicatedPartitionsMetricName, createLogReadResult, isListOffsetsTimestampUnsupported}
import kafka.server.metadata.{InklessMetadataView, KRaftMetadataCache}
import kafka.server.share.DelayedShareFetch
import kafka.utils._
Expand Down Expand Up @@ -160,6 +160,9 @@ object ReplicaManager {
private val IsrShrinksPerSecMetricName = "IsrShrinksPerSec"
private val FailedIsrUpdatesPerSecMetricName = "FailedIsrUpdatesPerSec"
private val ConsolidationFetchBytesInPerSecMetricName = "ConsolidationFetchBytesInPerSec"
private val ConsolidationSupplementRateMetricName = "ConsolidationSupplementRate"
private val ConsolidationSupplementBytesPerSecMetricName = "ConsolidationSupplementBytesPerSec"
private val ConsolidationLocalBytesPerSecMetricName = "ConsolidationLocalBytesPerSec"

private[server] val GaugeMetricNames = Set(
LeaderCountMetricName,
Expand All @@ -178,7 +181,10 @@ object ReplicaManager {
IsrExpandsPerSecMetricName,
IsrShrinksPerSecMetricName,
FailedIsrUpdatesPerSecMetricName,
ConsolidationFetchBytesInPerSecMetricName
ConsolidationFetchBytesInPerSecMetricName,
ConsolidationSupplementRateMetricName,
ConsolidationSupplementBytesPerSecMetricName,
ConsolidationLocalBytesPerSecMetricName
)

private[server] val MetricNames = GaugeMetricNames.union(MeterMetricNames)
Expand Down Expand Up @@ -449,6 +455,9 @@ class ReplicaManager(val config: KafkaConfig,
val isrShrinkRate: Meter = metricsGroup.newMeter(IsrShrinksPerSecMetricName, "shrinks", TimeUnit.SECONDS)
val failedIsrUpdatesRate: Meter = metricsGroup.newMeter(FailedIsrUpdatesPerSecMetricName, "failedUpdates", TimeUnit.SECONDS)
private val consolidationFetchBytesInPerSec: Meter = metricsGroup.newMeter(ConsolidationFetchBytesInPerSecMetricName, "bytes", TimeUnit.SECONDS)
private val consolidationSupplementRate: Meter = metricsGroup.newMeter(ConsolidationSupplementRateMetricName, "supplements", TimeUnit.SECONDS)
private val consolidationSupplementBytesPerSec: Meter = metricsGroup.newMeter(ConsolidationSupplementBytesPerSecMetricName, "bytes", TimeUnit.SECONDS)
private val consolidationLocalBytesPerSec: Meter = metricsGroup.newMeter(ConsolidationLocalBytesPerSecMetricName, "bytes", TimeUnit.SECONDS)

private def isConsolidatingPartition(partition: Partition): Boolean =
config.disklessRemoteStorageConsolidationEnabled && _inklessMetadataView.isConsolidatingDisklessTopic(partition.topic)
Expand Down Expand Up @@ -2301,6 +2310,13 @@ class ReplicaManager(val config: KafkaConfig,
* local read has not yet reached the local log end offset (the classic->diskless seal), are
* dropped — see the exhaustion guard below.
*/
private[server] def recordConsolidationSupplementIssued(): Unit = consolidationSupplementRate.mark()
private[server] def recordConsolidationSupplementBytes(bytes: Long): Unit = consolidationSupplementBytesPerSec.mark(bytes)
private[server] def recordConsolidationLocalBytes(bytes: Long): Unit = consolidationLocalBytesPerSec.mark(bytes)

private[server] def successfulSupplementBytes(data: Map[TopicIdPartition, FetchPartitionData]): Long =
data.values.filter(_.error == Errors.NONE).map(_.records.sizeInBytes.toLong).sum

private[server] def buildConsolidationSupplementFetchInfos(
supplements: Map[TopicIdPartition, Long],
fetchInfos: Seq[(TopicIdPartition, PartitionData)],
Expand Down Expand Up @@ -2656,6 +2672,7 @@ class ReplicaManager(val config: KafkaConfig,
var hasPreferredReadReplica = false
val logReadResultMap = new util.LinkedHashMap[TopicIdPartition, LogReadResult]

var consolidationLocalBytes = 0L
logReadResults.foreach { case (topicIdPartition, logReadResult) =>
brokerTopicStats.topicStats(topicIdPartition.topicPartition.topic).totalFetchRequestRate.mark()
brokerTopicStats.allTopicsStats.totalFetchRequestRate.mark()
Expand All @@ -2669,8 +2686,22 @@ class ReplicaManager(val config: KafkaConfig,
if (logReadResult.preferredReadReplica.isPresent)
hasPreferredReadReplica = true
bytesReadable = bytesReadable + logReadResult.info.records.sizeInBytes
if (consolidatingLocalFetchSupplements.contains(topicIdPartition)) {
consolidationLocalBytes += logReadResult.info.records.sizeInBytes
}
logReadResultMap.put(topicIdPartition, logReadResult)
}
// Record consolidation byte/rate metrics only on the paths that respond from this synchronous
// read. The delayedResponse path discards this read and re-reads (and re-issues the supplement)
// inside DelayedFetch, which records there; marking here as well would double-count
// (DelayedFetch.scala).
var consolidationSupplementIssued = false
var consolidationSupplementBytes = 0L
def recordConsolidationMetricsIfAny(): Unit = {
if (consolidationLocalBytes > 0) consolidationLocalBytesPerSec.mark(consolidationLocalBytes)
if (consolidationSupplementIssued) consolidationSupplementRate.mark()
if (consolidationSupplementBytes > 0) consolidationSupplementBytesPerSec.mark(consolidationSupplementBytes)
}

// For consolidating partitions where local log was read, supplement with diskless data if minBytes not satisfied.
// Only runs when there are no pure-diskless partitions in the request: if disklessFetchInfos is non-empty the
Expand All @@ -2687,14 +2718,16 @@ class ReplicaManager(val config: KafkaConfig,
val supplementFetchInfos = buildConsolidationSupplementFetchInfos(consolidatingLocalFetchSupplements, fetchInfos, logReadResultMap)

if (supplementFetchInfos.nonEmpty) {
consolidationSupplementIssued = true
try {
val supplementParams = fetchParamsWithNewMaxBytes(params, supplementFetchInfos.size.toFloat / fetchInfos.size.toFloat)
// Future not cancelled on failure — diskless reads are idempotent and hold no resources.
consolidationSupplementData = fetchDisklessMessages(supplementParams, supplementFetchInfos)
.get(Math.max(config.disklessFetchMaxWaitMs.toLong, params.maxWaitMs), TimeUnit.MILLISECONDS)
.toMap
bytesReadable += consolidationSupplementData.values
.filter(_.error == Errors.NONE).map(_.records.sizeInBytes).sum
val supplementBytes = successfulSupplementBytes(consolidationSupplementData)
consolidationSupplementBytes = supplementBytes
bytesReadable += supplementBytes
} catch {
case e: InterruptedException =>
Thread.currentThread().interrupt()
Expand Down Expand Up @@ -2732,6 +2765,7 @@ class ReplicaManager(val config: KafkaConfig,
// 6) has a preferred read replica
if (remoteFetchInfos.isEmpty && disklessFetchInfos.isEmpty && (params.maxWaitMs <= 0 || bytesReadable >= params.minBytes || errorReadingData ||
hasDivergingEpoch || hasPreferredReadReplica)) {
recordConsolidationMetricsIfAny()
respond(fetchPartitionData)
} else {
// construct the fetch results from the read results
Expand Down Expand Up @@ -2776,9 +2810,11 @@ class ReplicaManager(val config: KafkaConfig,
}
}
logReadResultMap.putAll(disklessFetchResults)
recordConsolidationMetricsIfAny()
processRemoteFetches(remoteFetchInfos, params, respond, logReadResultMap, fetchPartitionStatus)
} else {
if (disklessFetchInfos.isEmpty && (bytesReadable >= params.minBytes || params.maxWaitMs <= 0)) {
recordConsolidationMetricsIfAny()
respond(fetchPartitionData)
} else {
delayedResponse(fetchPartitionStatus)
Expand Down
Loading
Loading