Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions INKLESS_OWNERSHIP
Original file line number Diff line number Diff line change
Expand Up @@ -62,3 +62,5 @@ core/src/main/scala/kafka/server/DynamicBrokerConfig.scala @aiven/inkless @apach
core/src/main/scala/kafka/server/ConfigHandler.scala @aiven/inkless @apache/kafka # topic-config propagation
core/src/main/scala/kafka/server/ControllerServer.scala @aiven/inkless @apache/kafka # controller-side diskless wiring
core/src/main/scala/kafka/server/AbstractFetcherThread.scala @aiven/inkless @apache/kafka # consolidation fetcher skips ConsumerLag JMX registration
metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java @aiven/inkless @apache/kafka # diskless placement, election, and classic-to-diskless switch commits
metadata/src/main/java/org/apache/kafka/controller/PartitionChangeBuilder.java @aiven/inkless @apache/kafka # classic-to-diskless switch flag commit
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,8 @@ default DescribeTopicPartitionsResult describeTopicPartitions(Collection<String>
/**
* Describe topic partitions in the cluster, returning raw response data including unknown tagged fields.
* This is useful for accessing Inkless-specific metadata (classicToDisklessStartOffset, disklessLeaderEpoch,
* producerStates) that are encoded as tagged fields on the DescribeTopicPartitions response.
* producerStates, classicPrefixInRemoteStorage) that are encoded as tagged fields on the DescribeTopicPartitions
* response.
*
* @param topics The topics to describe.
* @param options The options to use when describing the topics.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,8 @@ public enum ApiKeys {
DELETE_SHARE_GROUP_OFFSETS(ApiMessageType.DELETE_SHARE_GROUP_OFFSETS),
INIT_DISKLESS_LOG(ApiMessageType.INIT_DISKLESS_LOG, true),
ALTER_DISKLESS_SWITCH(ApiMessageType.ALTER_DISKLESS_SWITCH, false, true),
REPAIR_DISKLESS_LOG(ApiMessageType.REPAIR_DISKLESS_LOG, false, false);
REPAIR_DISKLESS_LOG(ApiMessageType.REPAIR_DISKLESS_LOG, false, false),
SET_CLASSIC_PREFIX_IN_REMOTE_STORAGE(ApiMessageType.SET_CLASSIC_PREFIX_IN_REMOTE_STORAGE, true);


private static final Map<ApiMessageType.ListenerType, EnumSet<ApiKeys>> APIS_BY_LISTENER =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,8 @@ private static AbstractRequest doParseRequest(ApiKeys apiKey, short apiVersion,
return AlterDisklessSwitchRequest.parse(readable, apiVersion);
case REPAIR_DISKLESS_LOG:
return RepairDisklessLogRequest.parse(readable, apiVersion);
case SET_CLASSIC_PREFIX_IN_REMOTE_STORAGE:
return SetClassicPrefixInRemoteStorageRequest.parse(readable, apiVersion);
default:
throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseRequest`, the " +
"code should be updated to do so.", apiKey));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,8 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, Readable readable,
return AlterDisklessSwitchResponse.parse(readable, version);
case REPAIR_DISKLESS_LOG:
return RepairDisklessLogResponse.parse(readable, version);
case SET_CLASSIC_PREFIX_IN_REMOTE_STORAGE:
return SetClassicPrefixInRemoteStorageResponse.parse(readable, version);
default:
throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseResponse`, the " +
"code should be updated to do so.", apiKey));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.kafka.common.requests;

import org.apache.kafka.common.message.SetClassicPrefixInRemoteStorageRequestData;
import org.apache.kafka.common.message.SetClassicPrefixInRemoteStorageResponseData;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
import org.apache.kafka.common.protocol.Readable;

import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;

public class SetClassicPrefixInRemoteStorageRequest extends AbstractRequest {

private final SetClassicPrefixInRemoteStorageRequestData data;

public SetClassicPrefixInRemoteStorageRequest(SetClassicPrefixInRemoteStorageRequestData data, short apiVersion) {
super(ApiKeys.SET_CLASSIC_PREFIX_IN_REMOTE_STORAGE, apiVersion);
this.data = data;
}

@Override
public SetClassicPrefixInRemoteStorageRequestData data() {
return data;
}

@Override
public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) {
List<SetClassicPrefixInRemoteStorageResponseData.TopicResponse> results = new ArrayList<>();
data.topics().forEach(
topicData -> results.add(new SetClassicPrefixInRemoteStorageResponseData.TopicResponse()
.setTopicId(topicData.topicId())
.setPartitions(topicData.partitions().stream()
.map(partitionData -> new SetClassicPrefixInRemoteStorageResponseData.PartitionResponse()
.setPartitionId(partitionData.partitionId())
.setErrorCode(Errors.forException(e).code()))
.collect(Collectors.toList()))));
return new SetClassicPrefixInRemoteStorageResponse(new SetClassicPrefixInRemoteStorageResponseData()
.setThrottleTimeMs(throttleTimeMs)
.setTopics(results));
}

public static SetClassicPrefixInRemoteStorageRequest parse(Readable readable, short version) {
return new SetClassicPrefixInRemoteStorageRequest(new SetClassicPrefixInRemoteStorageRequestData(readable, version), version);
}

public static class Builder extends AbstractRequest.Builder<SetClassicPrefixInRemoteStorageRequest> {

private final SetClassicPrefixInRemoteStorageRequestData data;

public Builder(SetClassicPrefixInRemoteStorageRequestData data) {
super(
ApiKeys.SET_CLASSIC_PREFIX_IN_REMOTE_STORAGE,
ApiKeys.SET_CLASSIC_PREFIX_IN_REMOTE_STORAGE.oldestVersion(),
ApiKeys.SET_CLASSIC_PREFIX_IN_REMOTE_STORAGE.latestVersion()
);
this.data = data;
}

@Override
public SetClassicPrefixInRemoteStorageRequest build(short version) {
return new SetClassicPrefixInRemoteStorageRequest(data, version);
}

@Override
public String toString() {
return data.toString();
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.kafka.common.requests;

import org.apache.kafka.common.message.SetClassicPrefixInRemoteStorageResponseData;
import org.apache.kafka.common.protocol.ApiKeys;
import org.apache.kafka.common.protocol.Errors;
import org.apache.kafka.common.protocol.Readable;

import java.util.EnumMap;
import java.util.Map;

public class SetClassicPrefixInRemoteStorageResponse extends AbstractResponse {

private final SetClassicPrefixInRemoteStorageResponseData data;

public SetClassicPrefixInRemoteStorageResponse(SetClassicPrefixInRemoteStorageResponseData data) {
super(ApiKeys.SET_CLASSIC_PREFIX_IN_REMOTE_STORAGE);
this.data = data;
}

@Override
public SetClassicPrefixInRemoteStorageResponseData data() {
return data;
}

@Override
public Map<Errors, Integer> errorCounts() {
Map<Errors, Integer> counts = new EnumMap<>(Errors.class);
data.topics().forEach(topicResponse -> topicResponse.partitions().forEach(partitionResponse ->
updateErrorCounts(counts, Errors.forCode(partitionResponse.errorCode()))
));
return counts;
}

@Override
public int throttleTimeMs() {
return data.throttleTimeMs();
}

@Override
public void maybeSetThrottleTimeMs(int throttleTimeMs) {
data.setThrottleTimeMs(throttleTimeMs);
}

public static SetClassicPrefixInRemoteStorageResponse parse(Readable readable, short version) {
return new SetClassicPrefixInRemoteStorageResponse(new SetClassicPrefixInRemoteStorageResponseData(readable, version));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

{
"apiKey": 503,
"type": "request",
"listeners": ["controller"],
"name": "SetClassicPrefixInRemoteStorageRequest",
"validVersions": "0",
"flexibleVersions": "0+",
"fields": [
{ "name": "BrokerId", "type": "int32", "versions": "0+", "entityType": "brokerId",
"about": "The broker ID of the requesting leader." },
{ "name": "BrokerEpoch", "type": "int64", "versions": "0+", "default": "-1",
"about": "The epoch of the requesting broker." },
{ "name": "Topics", "type": "[]TopicData", "versions": "0+",
"about": "The topics whose classic prefix is in remote storage.", "fields": [
{ "name": "TopicId", "type": "uuid", "versions": "0+",
"about": "The topic ID." },
{ "name": "Partitions", "type": "[]PartitionData", "versions": "0+",
"about": "The partitions whose classic prefix is in remote storage.", "fields": [
{ "name": "PartitionId", "type": "int32", "versions": "0+",
"about": "The partition ID." },
{ "name": "ClassicToDisklessStartOffset", "type": "int64", "versions": "0+",
"about": "The committed seal: the first offset in diskless format." },
{ "name": "LeaderEpoch", "type": "int32", "versions": "0+",
"about": "The leader epoch seen by the broker." }
]}
]}
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

{
"apiKey": 503,
"type": "response",
"name": "SetClassicPrefixInRemoteStorageResponse",
"validVersions": "0",
"flexibleVersions": "0+",
"fields": [
{ "name": "ThrottleTimeMs", "type": "int32", "versions": "0+",
"about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." },
{ "name": "Topics", "type": "[]TopicResponse", "versions": "0+",
"about": "Per-topic results.", "fields": [
{ "name": "TopicId", "type": "uuid", "versions": "0+",
"about": "The topic ID." },
{ "name": "Partitions", "type": "[]PartitionResponse", "versions": "0+",
"about": "Per-partition results.", "fields": [
{ "name": "PartitionId", "type": "int32", "versions": "0+",
"about": "The partition ID." },
{ "name": "ErrorCode", "type": "int16", "versions": "0+",
"about": "The partition-level error code." }
]}
]}
]
}
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,8 @@
import org.apache.kafka.common.message.SaslAuthenticateResponseData;
import org.apache.kafka.common.message.SaslHandshakeRequestData;
import org.apache.kafka.common.message.SaslHandshakeResponseData;
import org.apache.kafka.common.message.SetClassicPrefixInRemoteStorageRequestData;
import org.apache.kafka.common.message.SetClassicPrefixInRemoteStorageResponseData;
import org.apache.kafka.common.message.ShareAcknowledgeRequestData;
import org.apache.kafka.common.message.ShareAcknowledgeResponseData;
import org.apache.kafka.common.message.ShareFetchRequestData;
Expand Down Expand Up @@ -1084,6 +1086,7 @@ private AbstractRequest getRequest(ApiKeys apikey, short version) {
case INIT_DISKLESS_LOG: return createInitDisklessLogRequest(version);
case ALTER_DISKLESS_SWITCH: return createAlterDisklessSwitchRequest(version);
case REPAIR_DISKLESS_LOG: return createRepairDisklessLogRequest(version);
case SET_CLASSIC_PREFIX_IN_REMOTE_STORAGE: return createSetClassicPrefixInRemoteStorageRequest(version);
default: throw new IllegalArgumentException("Unknown API key " + apikey);
}
}
Expand Down Expand Up @@ -1182,6 +1185,7 @@ private AbstractResponse getResponse(ApiKeys apikey, short version) {
case INIT_DISKLESS_LOG: return createInitDisklessLogResponse();
case ALTER_DISKLESS_SWITCH: return createAlterDisklessSwitchResponse();
case REPAIR_DISKLESS_LOG: return createRepairDisklessLogResponse();
case SET_CLASSIC_PREFIX_IN_REMOTE_STORAGE: return createSetClassicPrefixInRemoteStorageResponse();
default: throw new IllegalArgumentException("Unknown API key " + apikey);
}
}
Expand Down Expand Up @@ -1343,6 +1347,38 @@ private RepairDisklessLogResponse createRepairDisklessLogResponse() {
.setErrorCode(Errors.NONE.code()));
}

private SetClassicPrefixInRemoteStorageRequest createSetClassicPrefixInRemoteStorageRequest(short version) {
SetClassicPrefixInRemoteStorageRequestData data = new SetClassicPrefixInRemoteStorageRequestData()
.setBrokerId(1)
.setBrokerEpoch(123L)
.setTopics(singletonList(
new SetClassicPrefixInRemoteStorageRequestData.TopicData()
.setTopicId(Uuid.fromString("qo0Pcp70TdGnAa7YKMKCqw"))
.setPartitions(singletonList(
new SetClassicPrefixInRemoteStorageRequestData.PartitionData()
.setPartitionId(0)
.setClassicToDisklessStartOffset(100L)
.setLeaderEpoch(5)
))
));
return new SetClassicPrefixInRemoteStorageRequest.Builder(data).build(version);
}

private SetClassicPrefixInRemoteStorageResponse createSetClassicPrefixInRemoteStorageResponse() {
SetClassicPrefixInRemoteStorageResponseData data = new SetClassicPrefixInRemoteStorageResponseData()
.setThrottleTimeMs(123)
.setTopics(singletonList(
new SetClassicPrefixInRemoteStorageResponseData.TopicResponse()
.setTopicId(Uuid.fromString("qo0Pcp70TdGnAa7YKMKCqw"))
.setPartitions(singletonList(
new SetClassicPrefixInRemoteStorageResponseData.PartitionResponse()
.setPartitionId(0)
.setErrorCode(Errors.NONE.code())
))
));
return new SetClassicPrefixInRemoteStorageResponse(data);
}

private DescribeTopicPartitionsRequest createDescribeTopicPartitionsRequest(short version) {
DescribeTopicPartitionsRequestData data = new DescribeTopicPartitionsRequestData()
.setTopics(singletonList(new DescribeTopicPartitionsRequestData.TopicRequest().setName("foo")))
Expand Down
16 changes: 16 additions & 0 deletions core/src/main/scala/kafka/server/ControllerApis.scala
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ class ControllerApis(
case ApiKeys.REMOVE_RAFT_VOTER => handleRemoveRaftVoter(request)
case ApiKeys.UPDATE_RAFT_VOTER => handleUpdateRaftVoter(request)
case ApiKeys.INIT_DISKLESS_LOG => handleInitDisklessLogRequest(request)
case ApiKeys.SET_CLASSIC_PREFIX_IN_REMOTE_STORAGE => handleSetClassicPrefixInRemoteStorageRequest(request)
case ApiKeys.ALTER_DISKLESS_SWITCH => handleAlterDisklessSwitchRequest(request)
case _ => throw new ApiException(s"Unsupported ApiKey ${request.context.header.apiKey}")
}
Expand Down Expand Up @@ -689,6 +690,21 @@ class ControllerApis(
}
}

def handleSetClassicPrefixInRemoteStorageRequest(request: RequestChannel.Request): CompletableFuture[Unit] = {
val setClassicPrefixInRemoteStorageRequest = request.body[SetClassicPrefixInRemoteStorageRequest]
val context = new ControllerRequestContext(request.context.header.data, request.context.principal, OptionalLong.empty())
authHelper.authorizeClusterOperation(request, CLUSTER_ACTION)
val future = controller.setClassicPrefixInRemoteStorage(context, setClassicPrefixInRemoteStorageRequest.data())
future.handle[Unit] { (result, exception) =>
val response = if (exception != null) {
setClassicPrefixInRemoteStorageRequest.getErrorResponse(exception)
} else {
new SetClassicPrefixInRemoteStorageResponse(result)
}
requestHelper.sendResponseExemptThrottle(request, response)
}
}

def handleAlterDisklessSwitchRequest(request: RequestChannel.Request): CompletableFuture[Unit] = {
val alterDisklessSwitchRequest = request.body[AlterDisklessSwitchRequest]
authHelper.authorizeClusterOperation(request, ALTER)
Expand Down
Loading
Loading