Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions docs/inkless/configs.rst
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,13 @@ Under ``inkless.``
* Default: io.aiven.inkless.storage_backend.in_memory.InMemoryStorage
* Importance: high

``client.az.listener.map``
Cluster-wide mapping from AZ-specific listener aliases to physical availability zone (rack) IDs, used for listener-based AZ routing of diskless topic metadata. Format is a comma-separated list of LISTENER=az pairs, for example SASL_SSL_US_EAST_1_AZ_1=use1-az1,SASL_SSL_US_EAST_1_AZ_2=use1-az2. Every broker should use the same mapping. Listener names are matched case-insensitively (normalized to upper case). Defaults to empty (feature disabled).

* Type: list
* Default: ""
* Importance: medium

``control.plane.batch.coalescing.enabled``
When true, contiguous same-partition batch runs within a single commit are collapsed into a single row in the batches table via commit_file_v2, reducing control-plane metadata growth for low-throughput producers. Set to true only after all brokers in the cluster have been upgraded to a version that supports reading coalesced rows. Defaults to false for safe rolling upgrades.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,13 @@
import org.apache.kafka.common.config.ConfigDef;
import org.apache.kafka.common.config.ConfigException;
import org.apache.kafka.common.metrics.Metrics;
import org.apache.kafka.common.network.ListenerName;

import java.lang.reflect.InvocationTargetException;
import java.time.Duration;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

import io.aiven.inkless.common.config.validators.Subclass;
Expand Down Expand Up @@ -231,6 +235,14 @@ public class InklessConfig extends AbstractConfig {
+ "Defaults to false for safe rolling upgrades.";
private static final boolean BATCH_COALESCING_ENABLED_DEFAULT = false;

public static final String CLIENT_AZ_LISTENER_MAP_CONFIG = "client.az.listener.map";
private static final String CLIENT_AZ_LISTENER_MAP_DOC = "Cluster-wide mapping from AZ-specific listener aliases to physical "
+ "availability zone (rack) IDs, used for listener-based AZ routing of diskless topic metadata. "
+ "Format is a comma-separated list of LISTENER=az pairs, for example "
+ "SASL_SSL_US_EAST_1_AZ_1=use1-az1,SASL_SSL_US_EAST_1_AZ_2=use1-az2. "
+ "Every broker should use the same mapping. Listener names are matched case-insensitively "
+ "(normalized to upper case). Defaults to empty (feature disabled).";

public static ConfigDef configDef() {
final ConfigDef configDef = new ConfigDef();

Expand Down Expand Up @@ -485,9 +497,47 @@ public static ConfigDef configDef() {
BATCH_COALESCING_ENABLED_DOC
);

configDef.define(
CLIENT_AZ_LISTENER_MAP_CONFIG,
ConfigDef.Type.LIST,
Collections.emptyList(),
ConfigDef.Importance.MEDIUM,
CLIENT_AZ_LISTENER_MAP_DOC
);

return configDef;
}

private static Map<String, String> parseClientAzListenerMap(final List<String> entries) {
final Map<String, String> result = new LinkedHashMap<>();
for (final String entry : entries) {
final int sep = entry.indexOf('=');
if (sep < 0) {
throw new ConfigException(
CLIENT_AZ_LISTENER_MAP_CONFIG, entry,
"Entry is missing the '=' separator between listener name and AZ.");
}
final String listener = entry.substring(0, sep).trim();
final String az = entry.substring(sep + 1).trim();
if (listener.isEmpty()) {
throw new ConfigException(
CLIENT_AZ_LISTENER_MAP_CONFIG, entry, "Listener name must not be empty.");
}
if (az.isEmpty()) {
throw new ConfigException(
CLIENT_AZ_LISTENER_MAP_CONFIG, entry, "AZ must not be empty.");
}
final String normalizedListener = ListenerName.normalised(listener).value();
if (result.containsKey(normalizedListener)) {
throw new ConfigException(
CLIENT_AZ_LISTENER_MAP_CONFIG, entry,
"Duplicate listener name '" + normalizedListener + "' (matching is case-insensitive).");
}
Comment thread
Copilot marked this conversation as resolved.
Resolved
result.put(normalizedListener, az);
}
return result;
}

public InklessConfig(final AbstractConfig config) {
this(config.originalsWithPrefix(InklessConfig.PREFIX));
}
Expand Down Expand Up @@ -573,6 +623,13 @@ private static ConfigDef validate(final Map<String, ?> props) {
);
}

// Parse the AZ listener map to trigger strict validation at startup.
// The parsed result is discarded here; the accessor re-parses on demand.
@SuppressWarnings("unchecked")
final List<String> azListenerEntries =
(List<String>) parsedProps.get(CLIENT_AZ_LISTENER_MAP_CONFIG);
parseClientAzListenerMap(azListenerEntries);

return configDef;
}

Expand Down Expand Up @@ -722,4 +779,9 @@ public Duration batchCoordinateCacheTtl() {
public long cacheMaxBytes() {
return getLong(CONSUME_CACHE_MAX_BYTES_CONFIG);
}

public Map<String, String> clientAzListenerMap() {
return Collections.unmodifiableMap(
parseClientAzListenerMap(getList(CLIENT_AZ_LISTENER_MAP_CONFIG)));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -665,4 +665,71 @@ void fullConfigWithLaggingConsumer() {
assertThat(config.fetchLaggingConsumerRequestRateLimit()).isEqualTo(300);
assertThat(config.maxBatchesPerEnforcementRequest()).isEqualTo(10);
}

@Test
void clientAzListenerMapParsesEntries() {
final Map<String, String> configs = new HashMap<>();
configs.put("inkless.client.az.listener.map",
"SASL_SSL_EU_WEST_1_AZ_1=euw1-az1,SASL_SSL_EU_WEST_1_AZ_2=euw1-az2");
final InklessConfig config = new InklessConfig(new AbstractConfig(new ConfigDef(), configs));
assertThat(config.clientAzListenerMap()).isEqualTo(Map.of(
"SASL_SSL_EU_WEST_1_AZ_1", "euw1-az1",
"SASL_SSL_EU_WEST_1_AZ_2", "euw1-az2"
));
}

@Test
void clientAzListenerMapNormalizesListenerToUpperCase() {
final Map<String, String> configs = new HashMap<>();
configs.put("inkless.client.az.listener.map", "sasl_ssl_us_east_1_az_1=use1-az1");
final InklessConfig config = new InklessConfig(new AbstractConfig(new ConfigDef(), configs));
assertThat(config.clientAzListenerMap()).isEqualTo(Map.of("SASL_SSL_US_EAST_1_AZ_1", "use1-az1"));
}

@Test
void clientAzListenerMapDefaultsToEmpty() {
final InklessConfig config = new InklessConfig(new AbstractConfig(new ConfigDef(), new HashMap<>()));
assertThat(config.clientAzListenerMap()).isEmpty();
}

private InklessConfig configWithAzMap(final String value) {
final Map<String, String> configs = new HashMap<>();
configs.put("inkless.client.az.listener.map", value);
return new InklessConfig(new AbstractConfig(new ConfigDef(), configs));
}

@Test
void clientAzListenerMapRejectsMissingSeparator() {
assertThatThrownBy(() -> configWithAzMap("SASL_SSL_AZ1"))
.isInstanceOf(ConfigException.class)
.hasMessageContaining("missing the '=' separator");
}

@Test
void clientAzListenerMapRejectsEmptyListener() {
assertThatThrownBy(() -> configWithAzMap("=euw1-az1"))
.isInstanceOf(ConfigException.class)
.hasMessageContaining("Listener name must not be empty");
}

@Test
void clientAzListenerMapRejectsEmptyAz() {
assertThatThrownBy(() -> configWithAzMap("SASL_SSL_AZ1="))
.isInstanceOf(ConfigException.class)
.hasMessageContaining("AZ must not be empty");
}

@Test
void clientAzListenerMapRejectsDuplicateListener() {
assertThatThrownBy(() -> configWithAzMap("SASL_SSL_AZ1=euw1-az1,SASL_SSL_AZ1=euw1-az2"))
.isInstanceOf(ConfigException.class)
.hasMessageContaining("Duplicate listener name");
}

@Test
void clientAzListenerMapRejectsCaseInsensitiveDuplicateListener() {
assertThatThrownBy(() -> configWithAzMap("SASL_SSL_AZ1=euw1-az1,sasl_ssl_az1=euw1-az2"))
.isInstanceOf(ConfigException.class)
.hasMessageContaining("Duplicate listener name");
}
}
Loading