diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageArrowPageSource.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageArrowPageSource.java index a3495aa1d532..e63e674dcd71 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageArrowPageSource.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageArrowPageSource.java @@ -31,7 +31,6 @@ import java.io.IOException; import java.io.UncheckedIOException; -import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; import java.util.concurrent.CompletableFuture; @@ -57,7 +56,7 @@ public class BigQueryStorageArrowPageSource private final BigQuerySplit split; private final PageBuilder pageBuilder; private final MemoryContext memoryContext; - private final Iterator responses; + private final ReadRowsHelper.ReadRowsIterator responses; private CompletableFuture nextResponse; private boolean finished; @@ -142,8 +141,9 @@ public SourcePage getNextSourcePage() @Override public void close() { - bigQueryArrowToPageConverter.close(); + responses.close(); nextResponse.cancel(true); + bigQueryArrowToPageConverter.close(); bigQueryReadClient.close(); } diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageAvroPageSource.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageAvroPageSource.java index ffe0d11889d7..81e8711e6653 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageAvroPageSource.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageAvroPageSource.java @@ -101,7 +101,7 @@ public class BigQueryStorageAvroPageSource private final AtomicLong readTimeNanos = new AtomicLong(); private final PageBuilder pageBuilder; private final MemoryContext memoryContext; - private final Iterator responses; + private final ReadRowsHelper.ReadRowsIterator responses; private CompletableFuture nextResponse; private boolean finished; @@ -339,6 +339,7 @@ private void writeRow(RowBlockBuilder output, RowType rowType, GenericRecord rec @Override public void close() { + responses.close(); nextResponse.cancel(true); bigQueryReadClient.close(); } diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/ReadRowsHelper.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/ReadRowsHelper.java index 8f7b7991f97f..6c6408972d97 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/ReadRowsHelper.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/ReadRowsHelper.java @@ -13,11 +13,13 @@ */ package io.trino.plugin.bigquery; +import com.google.api.gax.rpc.ServerStream; import com.google.cloud.bigquery.storage.v1.BigQueryReadClient; import com.google.cloud.bigquery.storage.v1.ReadRowsRequest; import com.google.cloud.bigquery.storage.v1.ReadRowsResponse; import io.airlift.log.Logger; +import java.io.Closeable; import java.util.Iterator; import java.util.NoSuchElementException; @@ -38,39 +40,40 @@ public ReadRowsHelper(BigQueryReadClient client, String streamName, int maxReadR this.maxReadRowsRetries = maxReadRowsRetries; } - public Iterator readRows() + ReadRowsIterator readRows() { - Iterator serverResponses = fetchResponses(0); - return new ReadRowsIterator(this, serverResponses); + return new ReadRowsIterator(this, fetchResponses(0)); } // In order to enable testing - protected Iterator fetchResponses(long offset) + protected ServerStream fetchResponses(long offset) { log.debug("Reading rows from %s offset %s", streamName, offset); return client.readRowsCallable() .call(ReadRowsRequest.newBuilder() .setReadStream(streamName) .setOffset(offset) - .build()) - .iterator(); + .build()); } // Ported from https://github.com/GoogleCloudDataproc/spark-bigquery-connector/pull/150 - private static class ReadRowsIterator - implements Iterator + static final class ReadRowsIterator + implements Iterator, Closeable { private final ReadRowsHelper helper; private long nextOffset; + private volatile ServerStream serverStream; private Iterator serverResponses; private int retries; + private volatile boolean closed; public ReadRowsIterator( ReadRowsHelper helper, - Iterator serverResponses) + ServerStream serverStream) { this.helper = helper; - this.serverResponses = serverResponses; + this.serverStream = serverStream; + this.serverResponses = serverStream.iterator(); } @Override @@ -93,7 +96,11 @@ public ReadRowsResponse next() // if relevant, retry the read, from the last read position if (BigQueryUtil.isRetryable(e) && retries < helper.maxReadRowsRetries) { log.debug("Request failed, retrying: %s", e); - serverResponses = helper.fetchResponses(nextOffset); + serverStream = helper.fetchResponses(nextOffset); + serverResponses = serverStream.iterator(); + if (closed) { + serverStream.cancel(); + } retries++; } else { @@ -106,5 +113,13 @@ public ReadRowsResponse next() throw new NoSuchElementException("No more server responses"); } + + @Override + public void close() + { + closed = true; + // Closing the client does not cancel in-flight calls, which keep holding read connection quota + serverStream.cancel(); + } } } diff --git a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSourceMemoryTracking.java b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSource.java similarity index 81% rename from plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSourceMemoryTracking.java rename to plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSource.java index 840e9e663c78..ae962910f9a5 100644 --- a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSourceMemoryTracking.java +++ b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSource.java @@ -30,6 +30,7 @@ import io.grpc.Server; import io.grpc.inprocess.InProcessChannelBuilder; import io.grpc.inprocess.InProcessServerBuilder; +import io.grpc.stub.ServerCallStreamObserver; import io.grpc.stub.StreamObserver; import io.trino.spi.PageBuilder; import io.trino.spi.TrinoException; @@ -64,13 +65,17 @@ import java.util.ArrayList; import java.util.List; import java.util.OptionalInt; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicInteger; import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; +import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.trino.spi.StandardErrorCode.EXCEEDED_LOCAL_MEMORY_LIMIT; import static io.trino.spi.type.BigintType.BIGINT; import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; +import static java.util.concurrent.Executors.newCachedThreadPool; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; @@ -78,7 +83,7 @@ @TestInstance(PER_CLASS) @Execution(CONCURRENT) -final class TestBigQueryStoragePageSourceMemoryTracking +final class TestBigQueryStoragePageSource { private static final BigQueryTypeManager TYPE_MANAGER = new BigQueryTypeManager(TESTING_TYPE_MANAGER); private static final List COLUMNS = ImmutableList.of( @@ -90,12 +95,15 @@ final class TestBigQueryStoragePageSourceMemoryTracking // Fetch the responses synchronously, so that every page is available when requested private final ExecutorService executor = newDirectExecutorService(); + // Fetch the responses in the background, so that a fetch can wait for a response that never comes + private final ExecutorService backgroundExecutor = newCachedThreadPool(daemonThreadsNamed("test-bigquery-read-%s")); private final BigQueryArrowBufferAllocator arrowBufferAllocator = new BigQueryArrowBufferAllocator(new BigQueryArrowConfig(), new BigQueryArrowAllocatorStats()); @AfterAll void tearDown() { executor.shutdownNow(); + backgroundExecutor.shutdownNow(); } @Test @@ -104,7 +112,7 @@ void testArrowMemoryUsage() { try (TestingBigQueryReadServer server = new TestingBigQueryReadServer(ImmutableList.of(arrowResponse(0, 100), arrowResponse(100, 50)))) { List reportedBytes = new ArrayList<>(); - assertMemoryUsage(createArrowPageSource(server.client(), reportedBytes::add), reportedBytes); + assertMemoryUsage(createArrowPageSource(server.client(), executor, reportedBytes::add), reportedBytes); } } @@ -114,7 +122,7 @@ void testAvroMemoryUsage() { try (TestingBigQueryReadServer server = new TestingBigQueryReadServer(ImmutableList.of(avroResponse(0, 100), avroResponse(100, 50)))) { List reportedBytes = new ArrayList<>(); - assertMemoryUsage(createAvroPageSource(server.client(), reportedBytes::add), reportedBytes); + assertMemoryUsage(createAvroPageSource(server.client(), executor, reportedBytes::add), reportedBytes); } } @@ -124,7 +132,7 @@ void testArrowMemoryLimitExceededOnCreation() { try (TestingBigQueryReadServer server = new TestingBigQueryReadServer(ImmutableList.of())) { TrinoException memoryLimitExceeded = new TrinoException(EXCEEDED_LOCAL_MEMORY_LIMIT, "Exceeded memory limit"); - assertThatThrownBy(() -> createArrowPageSource(server.client(), _ -> { + assertThatThrownBy(() -> createArrowPageSource(server.client(), executor, _ -> { throw memoryLimitExceeded; })).isSameAs(memoryLimitExceeded); assertThat(server.readRowsRequests()).isZero(); @@ -137,13 +145,41 @@ void testAvroMemoryLimitExceededOnCreation() { try (TestingBigQueryReadServer server = new TestingBigQueryReadServer(ImmutableList.of())) { TrinoException memoryLimitExceeded = new TrinoException(EXCEEDED_LOCAL_MEMORY_LIMIT, "Exceeded memory limit"); - assertThatThrownBy(() -> createAvroPageSource(server.client(), _ -> { + assertThatThrownBy(() -> createAvroPageSource(server.client(), executor, _ -> { throw memoryLimitExceeded; })).isSameAs(memoryLimitExceeded); assertThat(server.readRowsRequests()).isZero(); } } + @Test + void testArrowCloseCancelsUnfinishedStream() + throws Exception + { + try (TestingBigQueryReadServer server = new TestingBigQueryReadServer(ImmutableList.of(arrowResponse(0, 100)), false)) { + assertCloseCancelsUnfinishedStream(createArrowPageSource(server.client(), backgroundExecutor, _ -> {}), server); + } + } + + @Test + void testAvroCloseCancelsUnfinishedStream() + throws Exception + { + try (TestingBigQueryReadServer server = new TestingBigQueryReadServer(ImmutableList.of(avroResponse(0, 100)), false)) { + assertCloseCancelsUnfinishedStream(createAvroPageSource(server.client(), backgroundExecutor, _ -> {}), server); + } + } + + private static void assertCloseCancelsUnfinishedStream(ConnectorPageSource pageSource, TestingBigQueryReadServer server) + throws Exception + { + pageSource.isBlocked().get(10, SECONDS); + assertPage(pageSource.getNextSourcePage(), 0, 100); + + pageSource.close(); + assertThat(server.awaitReadRowsCancelled()).isTrue(); + } + private static void assertMemoryUsage(ConnectorPageSource pageSource, List reportedBytes) throws IOException { @@ -173,13 +209,13 @@ private static void assertPage(SourcePage page, long firstValue, int positionCou } } - private ConnectorPageSource createArrowPageSource(BigQueryReadClient client, MemoryContext memoryContext) + private ConnectorPageSource createArrowPageSource(BigQueryReadClient client, ExecutorService executor, MemoryContext memoryContext) { BigQuerySplit split = BigQuerySplit.forStream("trace", "stream", ARROW_SCHEMA.toJson(), COLUMNS, OptionalInt.empty()); return new BigQueryStorageArrowPageSource(TYPE_MANAGER, client, executor, arrowBufferAllocator, 0, split, COLUMNS, memoryContext); } - private ConnectorPageSource createAvroPageSource(BigQueryReadClient client, MemoryContext memoryContext) + private ConnectorPageSource createAvroPageSource(BigQueryReadClient client, ExecutorService executor, MemoryContext memoryContext) { BigQuerySplit split = BigQuerySplit.forStream("trace", "stream", AVRO_SCHEMA.toString(), COLUMNS, OptionalInt.empty()); return new BigQueryStorageAvroPageSource(client, executor, TYPE_MANAGER, 0, split, COLUMNS, memoryContext); @@ -230,12 +266,19 @@ private static class TestingBigQueryReadServer implements AutoCloseable { private final AtomicInteger readRowsRequests = new AtomicInteger(); + private final CountDownLatch readRowsCancelled = new CountDownLatch(1); private final Server server; private final ManagedChannel channel; private final BigQueryReadClient client; public TestingBigQueryReadServer(List responses) throws IOException + { + this(responses, true); + } + + public TestingBigQueryReadServer(List responses, boolean completeStream) + throws IOException { String serverName = InProcessServerBuilder.generateName(); server = InProcessServerBuilder.forName(serverName) @@ -246,8 +289,12 @@ public TestingBigQueryReadServer(List responses) public void readRows(ReadRowsRequest request, StreamObserver responseObserver) { readRowsRequests.incrementAndGet(); + ((ServerCallStreamObserver) responseObserver).setOnCancelHandler(readRowsCancelled::countDown); responses.forEach(responseObserver::onNext); - responseObserver.onCompleted(); + // Without completing, the stream stays open like a BigQuery stream with rows left to read + if (completeStream) { + responseObserver.onCompleted(); + } } }) .build() @@ -271,6 +318,12 @@ public int readRowsRequests() return readRowsRequests.get(); } + public boolean awaitReadRowsCancelled() + throws InterruptedException + { + return readRowsCancelled.await(10, SECONDS); + } + @Override public void close() {