Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions INKLESS_OWNERSHIP
Original file line number Diff line number Diff line change
Expand Up @@ -61,3 +61,4 @@ core/src/main/scala/kafka/server/DelayedFetch.scala @aiven/inkless @apach
core/src/main/scala/kafka/server/DynamicBrokerConfig.scala @aiven/inkless @apache/kafka # dynamic inkless config reconfig
core/src/main/scala/kafka/server/ConfigHandler.scala @aiven/inkless @apache/kafka # topic-config propagation
core/src/main/scala/kafka/server/ControllerServer.scala @aiven/inkless @apache/kafka # controller-side diskless wiring
core/src/main/scala/kafka/server/AbstractFetcherThread.scala @aiven/inkless @apache/kafka # consolidation fetcher skips ConsumerLag JMX registration
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ class ConsolidationFetcherThread(name: String,

override protected def shouldRecordReplicationBytesIn: Boolean = false

// `ConsolidationLocalLag` is the external first-hop lag signal. Keep updating
// `fetcherLagStats` for `MaxLag`, but don't register the generic `ConsumerLag` JMX gauge.
override protected def registerFetcherConsumerLagMetrics: Boolean = false

override def processPartitionData(
topicPartition: TopicPartition,
fetchOffset: Long,
Expand Down
22 changes: 16 additions & 6 deletions core/src/main/scala/kafka/server/AbstractFetcherThread.scala
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,12 @@ abstract class AbstractFetcherThread(name: String,

private val metricId = new ClientIdAndBroker(clientId, leader.brokerEndPoint().host, leader.brokerEndPoint().port)
val fetcherStats = new FetcherStats(metricId)
val fetcherLagStats = new FetcherLagStats(metricId)
val fetcherLagStats = new FetcherLagStats(metricId, registerFetcherConsumerLagMetrics)

// When false, keep updating in-memory fetcherLagStats for MaxLag but skip the
// kafka.server:FetcherLagMetrics/ConsumerLag JMX gauge. ConsolidationFetcherThread
// overrides this so ConsolidationLocalLag is the external consolidation lag signal.
protected def registerFetcherConsumerLagMetrics: Boolean = true

/* callbacks to be defined in subclass */

Expand Down Expand Up @@ -923,7 +928,7 @@ object FetcherMetrics {
val BytesPerSec = "BytesPerSec"
}

class FetcherLagMetrics(metricId: ClientIdTopicPartition) {
class FetcherLagMetrics(metricId: ClientIdTopicPartition, registerConsumerLag: Boolean = true) {
// Changing the package or class name may cause incompatibility with existing code and metrics configuration
private val metricsPackage = "kafka.server"
private val metricsClassName = "FetcherLagMetrics"
Expand All @@ -935,7 +940,9 @@ class FetcherLagMetrics(metricId: ClientIdTopicPartition) {
"topic" -> metricId.topicPartition.topic,
"partition" -> metricId.topicPartition.partition.toString).asJava

metricsGroup.newGauge(FetcherMetrics.ConsumerLag, () => lagVal.get, tags)
if (registerConsumerLag) {
metricsGroup.newGauge(FetcherMetrics.ConsumerLag, () => lagVal.get, tags)
}

def lag_=(newLag: Long): Unit = {
lagVal.set(newLag)
Expand All @@ -944,15 +951,18 @@ class FetcherLagMetrics(metricId: ClientIdTopicPartition) {
def lag: Long = lagVal.get

def unregister(): Unit = {
metricsGroup.removeMetric(FetcherMetrics.ConsumerLag, tags)
if (registerConsumerLag) {
metricsGroup.removeMetric(FetcherMetrics.ConsumerLag, tags)
}
}
}

class FetcherLagStats(metricId: ClientIdAndBroker) {
class FetcherLagStats(metricId: ClientIdAndBroker, registerConsumerLag: Boolean = true) {
val stats = new ConcurrentHashMap[TopicPartition, FetcherLagMetrics]

def getAndMaybePut(topicPartition: TopicPartition): FetcherLagMetrics = {
stats.computeIfAbsent(topicPartition, k => new FetcherLagMetrics(ClientIdTopicPartition(metricId.clientId, k)))
stats.computeIfAbsent(topicPartition, k =>
new FetcherLagMetrics(ClientIdTopicPartition(metricId.clientId, k), registerConsumerLag))
}

def unregister(topicPartition: TopicPartition): Unit = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,25 @@ class ConsolidationFetcherThreadTest {
assertEquals(50L, findGaugeValue("ConsolidationDeletableMessages", topicPartition))
}

@Test
def testTracksFetcherLagWithoutRegisteringConsumerLag(): Unit = {
val partition = mockPartitionWithLog(80L, 60L)
val replicaManager = mockReplicaManager(partition)
val thread = createConsolidationFetcherThread(replicaManager, Some(metrics))

val lagMetrics = thread.fetcherLagStats.getAndMaybePut(topicPartition)
lagMetrics.lag = 42L

assertEquals(42L, thread.fetcherLagStats.stats.get(topicPartition).lag)
assertFalse(
KafkaYammerMetrics.defaultRegistry.allMetrics.asScala.keys.exists { metricName =>
metricName.getGroup == "kafka.server" &&
metricName.getType == "FetcherLagMetrics" &&
metricName.getName == FetcherMetrics.ConsumerLag
},
"ConsolidationFetcherThread must not register the generic FetcherLagMetrics.ConsumerLag JMX gauge")
}

@Test
def testRecordsConsolidationFetchBytesInAndSkipsReplicationBytesIn(): Unit = {
val validBytes = 512
Expand Down
2 changes: 2 additions & 0 deletions docs/inkless/DISKLESS_CONSOLIDATION.md
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,8 @@ The broker registers these under the `io.aiven.inkless.consolidation` group. The
| `io.aiven.inkless.cache:type=CrossTierLogStartCache` | `CacheHits` / `CacheMisses` / `CacheSize` | Cross-tier earliest-offset cache. |
| controller | `DisklessWithoutRemoteStorageCount` | Diskless topics where `remote.storage.enable` is `false` or unset, including born-diskless topics that never enter the `Failed` reconciler state. |

Consolidation fetchers report first-hop lag through `ConsolidationLocalLag`. They do not contribute to Kafka's generic `kafka.server:type=FetcherLagMetrics,name=ConsumerLag` metric, so replica catch-up dashboards that sum that metric exclude consolidation backlog.

## Compatibility

- **Produce / consume / replication**: Standard Kafka client APIs. A consolidated topic behaves like a tiered topic for clients.
Expand Down
Loading