Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@

import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
import java.util.concurrent.CompletableFuture;
Expand All @@ -57,7 +56,7 @@ public class BigQueryStorageArrowPageSource
private final BigQuerySplit split;
private final PageBuilder pageBuilder;
private final MemoryContext memoryContext;
private final Iterator<ReadRowsResponse> responses;
private final ReadRowsHelper.ReadRowsIterator responses;

private CompletableFuture<ReadRowsResponse> nextResponse;
private boolean finished;
Expand Down Expand Up @@ -142,8 +141,9 @@ public SourcePage getNextSourcePage()
@Override
public void close()
{
bigQueryArrowToPageConverter.close();
responses.close();
nextResponse.cancel(true);
bigQueryArrowToPageConverter.close();
bigQueryReadClient.close();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ public class BigQueryStorageAvroPageSource
private final AtomicLong readTimeNanos = new AtomicLong();
private final PageBuilder pageBuilder;
private final MemoryContext memoryContext;
private final Iterator<ReadRowsResponse> responses;
private final ReadRowsHelper.ReadRowsIterator responses;

private CompletableFuture<ReadRowsResponse> nextResponse;
private boolean finished;
Expand Down Expand Up @@ -339,6 +339,7 @@ private void writeRow(RowBlockBuilder output, RowType rowType, GenericRecord rec
@Override
public void close()
{
responses.close();
nextResponse.cancel(true);
bigQueryReadClient.close();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,13 @@
*/
package io.trino.plugin.bigquery;

import com.google.api.gax.rpc.ServerStream;
import com.google.cloud.bigquery.storage.v1.BigQueryReadClient;
import com.google.cloud.bigquery.storage.v1.ReadRowsRequest;
import com.google.cloud.bigquery.storage.v1.ReadRowsResponse;
import io.airlift.log.Logger;

import java.io.Closeable;
import java.util.Iterator;
import java.util.NoSuchElementException;

Expand All @@ -38,39 +40,40 @@ public ReadRowsHelper(BigQueryReadClient client, String streamName, int maxReadR
this.maxReadRowsRetries = maxReadRowsRetries;
}

public Iterator<ReadRowsResponse> readRows()
ReadRowsIterator readRows()
{
Iterator<ReadRowsResponse> serverResponses = fetchResponses(0);
return new ReadRowsIterator(this, serverResponses);
return new ReadRowsIterator(this, fetchResponses(0));
}

// In order to enable testing
protected Iterator<ReadRowsResponse> fetchResponses(long offset)
protected ServerStream<ReadRowsResponse> fetchResponses(long offset)
{
log.debug("Reading rows from %s offset %s", streamName, offset);
return client.readRowsCallable()
.call(ReadRowsRequest.newBuilder()
.setReadStream(streamName)
.setOffset(offset)
.build())
.iterator();
.build());
}

// Ported from https://github.com/GoogleCloudDataproc/spark-bigquery-connector/pull/150
private static class ReadRowsIterator
implements Iterator<ReadRowsResponse>
static final class ReadRowsIterator
implements Iterator<ReadRowsResponse>, Closeable
{
private final ReadRowsHelper helper;
private long nextOffset;
private volatile ServerStream<ReadRowsResponse> serverStream;
private Iterator<ReadRowsResponse> serverResponses;
private int retries;
private volatile boolean closed;

public ReadRowsIterator(
ReadRowsHelper helper,
Iterator<ReadRowsResponse> serverResponses)
ServerStream<ReadRowsResponse> serverStream)
{
this.helper = helper;
this.serverResponses = serverResponses;
this.serverStream = serverStream;
this.serverResponses = serverStream.iterator();
}

@Override
Expand All @@ -93,7 +96,11 @@ public ReadRowsResponse next()
// if relevant, retry the read, from the last read position
if (BigQueryUtil.isRetryable(e) && retries < helper.maxReadRowsRetries) {
log.debug("Request failed, retrying: %s", e);
serverResponses = helper.fetchResponses(nextOffset);
serverStream = helper.fetchResponses(nextOffset);
serverResponses = serverStream.iterator();
if (closed) {
serverStream.cancel();
}
retries++;
}
else {
Expand All @@ -106,5 +113,13 @@ public ReadRowsResponse next()

throw new NoSuchElementException("No more server responses");
}

@Override
public void close()
{
closed = true;
// Closing the client does not cancel in-flight calls, which keep holding read connection quota
serverStream.cancel();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import io.grpc.Server;
import io.grpc.inprocess.InProcessChannelBuilder;
import io.grpc.inprocess.InProcessServerBuilder;
import io.grpc.stub.ServerCallStreamObserver;
import io.grpc.stub.StreamObserver;
import io.trino.spi.PageBuilder;
import io.trino.spi.TrinoException;
Expand Down Expand Up @@ -64,21 +65,25 @@
import java.util.ArrayList;
import java.util.List;
import java.util.OptionalInt;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicInteger;

import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService;
import static io.airlift.concurrent.Threads.daemonThreadsNamed;
import static io.trino.spi.StandardErrorCode.EXCEEDED_LOCAL_MEMORY_LIMIT;
import static io.trino.spi.type.BigintType.BIGINT;
import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER;
import static java.util.concurrent.Executors.newCachedThreadPool;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS;
import static org.junit.jupiter.api.parallel.ExecutionMode.CONCURRENT;

@TestInstance(PER_CLASS)
@Execution(CONCURRENT)
final class TestBigQueryStoragePageSourceMemoryTracking
final class TestBigQueryStoragePageSource
{
private static final BigQueryTypeManager TYPE_MANAGER = new BigQueryTypeManager(TESTING_TYPE_MANAGER);
private static final List<BigQueryColumnHandle> COLUMNS = ImmutableList.of(
Expand All @@ -90,12 +95,15 @@

// Fetch the responses synchronously, so that every page is available when requested
private final ExecutorService executor = newDirectExecutorService();
// Fetch the responses in the background, so that a fetch can wait for a response that never comes
private final ExecutorService backgroundExecutor = newCachedThreadPool(daemonThreadsNamed("test-bigquery-read-%s"));
private final BigQueryArrowBufferAllocator arrowBufferAllocator = new BigQueryArrowBufferAllocator(new BigQueryArrowConfig(), new BigQueryArrowAllocatorStats());

@AfterAll
void tearDown()
{
executor.shutdownNow();
backgroundExecutor.shutdownNow();
}

@Test
Expand All @@ -104,7 +112,7 @@
{
try (TestingBigQueryReadServer server = new TestingBigQueryReadServer(ImmutableList.of(arrowResponse(0, 100), arrowResponse(100, 50)))) {
List<Long> reportedBytes = new ArrayList<>();
assertMemoryUsage(createArrowPageSource(server.client(), reportedBytes::add), reportedBytes);
assertMemoryUsage(createArrowPageSource(server.client(), executor, reportedBytes::add), reportedBytes);
}
}

Expand All @@ -114,7 +122,7 @@
{
try (TestingBigQueryReadServer server = new TestingBigQueryReadServer(ImmutableList.of(avroResponse(0, 100), avroResponse(100, 50)))) {
List<Long> reportedBytes = new ArrayList<>();
assertMemoryUsage(createAvroPageSource(server.client(), reportedBytes::add), reportedBytes);
assertMemoryUsage(createAvroPageSource(server.client(), executor, reportedBytes::add), reportedBytes);
}
}

Expand All @@ -124,7 +132,7 @@
{
try (TestingBigQueryReadServer server = new TestingBigQueryReadServer(ImmutableList.of())) {
TrinoException memoryLimitExceeded = new TrinoException(EXCEEDED_LOCAL_MEMORY_LIMIT, "Exceeded memory limit");
assertThatThrownBy(() -> createArrowPageSource(server.client(), _ -> {
assertThatThrownBy(() -> createArrowPageSource(server.client(), executor, _ -> {
throw memoryLimitExceeded;
})).isSameAs(memoryLimitExceeded);
assertThat(server.readRowsRequests()).isZero();
Expand All @@ -137,13 +145,41 @@
{
try (TestingBigQueryReadServer server = new TestingBigQueryReadServer(ImmutableList.of())) {
TrinoException memoryLimitExceeded = new TrinoException(EXCEEDED_LOCAL_MEMORY_LIMIT, "Exceeded memory limit");
assertThatThrownBy(() -> createAvroPageSource(server.client(), _ -> {
assertThatThrownBy(() -> createAvroPageSource(server.client(), executor, _ -> {
throw memoryLimitExceeded;
})).isSameAs(memoryLimitExceeded);
assertThat(server.readRowsRequests()).isZero();
}
}

@Test
void testArrowCloseCancelsUnfinishedStream()
throws Exception
{
try (TestingBigQueryReadServer server = new TestingBigQueryReadServer(ImmutableList.of(arrowResponse(0, 100)), false)) {
assertCloseCancelsUnfinishedStream(createArrowPageSource(server.client(), backgroundExecutor, _ -> {}), server);
}
}

@Test
void testAvroCloseCancelsUnfinishedStream()
throws Exception
{
try (TestingBigQueryReadServer server = new TestingBigQueryReadServer(ImmutableList.of(avroResponse(0, 100)), false)) {
assertCloseCancelsUnfinishedStream(createAvroPageSource(server.client(), backgroundExecutor, _ -> {}), server);
}
}

private static void assertCloseCancelsUnfinishedStream(ConnectorPageSource pageSource, TestingBigQueryReadServer server)
throws Exception
{
pageSource.isBlocked().get(10, SECONDS);
assertPage(pageSource.getNextSourcePage(), 0, 100);

pageSource.close();
assertThat(server.awaitReadRowsCancelled()).isTrue();
}

private static void assertMemoryUsage(ConnectorPageSource pageSource, List<Long> reportedBytes)
throws IOException
{
Expand Down Expand Up @@ -173,13 +209,13 @@
}
}

private ConnectorPageSource createArrowPageSource(BigQueryReadClient client, MemoryContext memoryContext)
private ConnectorPageSource createArrowPageSource(BigQueryReadClient client, ExecutorService executor, MemoryContext memoryContext)
{
BigQuerySplit split = BigQuerySplit.forStream("trace", "stream", ARROW_SCHEMA.toJson(), COLUMNS, OptionalInt.empty());
return new BigQueryStorageArrowPageSource(TYPE_MANAGER, client, executor, arrowBufferAllocator, 0, split, COLUMNS, memoryContext);
}

private ConnectorPageSource createAvroPageSource(BigQueryReadClient client, MemoryContext memoryContext)
private ConnectorPageSource createAvroPageSource(BigQueryReadClient client, ExecutorService executor, MemoryContext memoryContext)
{
BigQuerySplit split = BigQuerySplit.forStream("trace", "stream", AVRO_SCHEMA.toString(), COLUMNS, OptionalInt.empty());
return new BigQueryStorageAvroPageSource(client, executor, TYPE_MANAGER, 0, split, COLUMNS, memoryContext);
Expand Down Expand Up @@ -230,12 +266,19 @@
implements AutoCloseable
{
private final AtomicInteger readRowsRequests = new AtomicInteger();
private final CountDownLatch readRowsCancelled = new CountDownLatch(1);
private final Server server;
private final ManagedChannel channel;
private final BigQueryReadClient client;

public TestingBigQueryReadServer(List<ReadRowsResponse> responses)

Check warning on line 274 in plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSource.java

View workflow job for this annotation

GitHub Actions / error-prone-checks

[EffectivelyPrivate] This declaration has public or protected modifiers, but is effectively private.
throws IOException
{
this(responses, true);
}

public TestingBigQueryReadServer(List<ReadRowsResponse> responses, boolean completeStream)

Check warning on line 280 in plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSource.java

View workflow job for this annotation

GitHub Actions / error-prone-checks

[EffectivelyPrivate] This declaration has public or protected modifiers, but is effectively private.
throws IOException
{
String serverName = InProcessServerBuilder.generateName();
server = InProcessServerBuilder.forName(serverName)
Expand All @@ -246,8 +289,12 @@
public void readRows(ReadRowsRequest request, StreamObserver<ReadRowsResponse> responseObserver)
{
readRowsRequests.incrementAndGet();
((ServerCallStreamObserver<ReadRowsResponse>) responseObserver).setOnCancelHandler(readRowsCancelled::countDown);
responses.forEach(responseObserver::onNext);
responseObserver.onCompleted();
// Without completing, the stream stays open like a BigQuery stream with rows left to read
if (completeStream) {
responseObserver.onCompleted();
}
}
})
.build()
Expand All @@ -261,16 +308,22 @@
.build());
}

public BigQueryReadClient client()

Check warning on line 311 in plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSource.java

View workflow job for this annotation

GitHub Actions / error-prone-checks

[EffectivelyPrivate] This declaration has public or protected modifiers, but is effectively private.
{
return client;
}

public int readRowsRequests()

Check warning on line 316 in plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSource.java

View workflow job for this annotation

GitHub Actions / error-prone-checks

[EffectivelyPrivate] This declaration has public or protected modifiers, but is effectively private.
{
return readRowsRequests.get();
}

public boolean awaitReadRowsCancelled()

Check warning on line 321 in plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSource.java

View workflow job for this annotation

GitHub Actions / error-prone-checks

[EffectivelyPrivate] This declaration has public or protected modifiers, but is effectively private.
throws InterruptedException
{
return readRowsCancelled.await(10, SECONDS);
}

@Override
public void close()
{
Expand Down
Loading