From 3d3774127a3d44e2572ed6ee4318599c3a09ef2a Mon Sep 17 00:00:00 2001 From: akmhmgc Date: Sun, 1 Nov 2026 14:23:58 +0900 Subject: [PATCH 1/2] Rename BigQuery storage page source test --- ...ceMemoryTracking.java => TestBigQueryStoragePageSource.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/{TestBigQueryStoragePageSourceMemoryTracking.java => TestBigQueryStoragePageSource.java} (99%) diff --git a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSourceMemoryTracking.java b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSource.java similarity index 99% rename from plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSourceMemoryTracking.java rename to plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSource.java index 840e9e663c78..55dc618d251b 100644 --- a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSourceMemoryTracking.java +++ b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSource.java @@ -78,7 +78,7 @@ @TestInstance(PER_CLASS) @Execution(CONCURRENT) -final class TestBigQueryStoragePageSourceMemoryTracking +final class TestBigQueryStoragePageSource { private static final BigQueryTypeManager TYPE_MANAGER = new BigQueryTypeManager(TESTING_TYPE_MANAGER); private static final List COLUMNS = ImmutableList.of( From c57fe32d8f13f06ec2c544eeb9c57c49b6d3f505 Mon Sep 17 00:00:00 2001 From: akmhmgc Date: Sun, 1 Nov 2026 14:26:17 +0900 Subject: [PATCH 2/2] Cancel unfinished BigQuery read streams on close Closing the BigQueryReadClient only shuts the gRPC channel down gracefully, which lets in-flight calls continue, and cancelling the future that fetches the next response does not stop the call either. When splits are closed before their streams end, because the query failed, was cancelled, or finished early, the ReadRows streams stay open on the BigQuery side. Each one counts against the concurrent read connections quota until BigQuery ends it, so later reads in the same project fail with RESOURCE_EXHAUSTED. Keep the current ServerStream in ReadRowsHelper and cancel it first when the page source is closed. --- .../BigQueryStorageArrowPageSource.java | 6 +- .../BigQueryStorageAvroPageSource.java | 3 +- .../trino/plugin/bigquery/ReadRowsHelper.java | 37 +++++++--- .../TestBigQueryStoragePageSource.java | 67 +++++++++++++++++-- 4 files changed, 91 insertions(+), 22 deletions(-) diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageArrowPageSource.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageArrowPageSource.java index a3495aa1d532..e63e674dcd71 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageArrowPageSource.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageArrowPageSource.java @@ -31,7 +31,6 @@ import java.io.IOException; import java.io.UncheckedIOException; -import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; import java.util.concurrent.CompletableFuture; @@ -57,7 +56,7 @@ public class BigQueryStorageArrowPageSource private final BigQuerySplit split; private final PageBuilder pageBuilder; private final MemoryContext memoryContext; - private final Iterator responses; + private final ReadRowsHelper.ReadRowsIterator responses; private CompletableFuture nextResponse; private boolean finished; @@ -142,8 +141,9 @@ public SourcePage getNextSourcePage() @Override public void close() { - bigQueryArrowToPageConverter.close(); + responses.close(); nextResponse.cancel(true); + bigQueryArrowToPageConverter.close(); bigQueryReadClient.close(); } diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageAvroPageSource.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageAvroPageSource.java index ffe0d11889d7..81e8711e6653 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageAvroPageSource.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/BigQueryStorageAvroPageSource.java @@ -101,7 +101,7 @@ public class BigQueryStorageAvroPageSource private final AtomicLong readTimeNanos = new AtomicLong(); private final PageBuilder pageBuilder; private final MemoryContext memoryContext; - private final Iterator responses; + private final ReadRowsHelper.ReadRowsIterator responses; private CompletableFuture nextResponse; private boolean finished; @@ -339,6 +339,7 @@ private void writeRow(RowBlockBuilder output, RowType rowType, GenericRecord rec @Override public void close() { + responses.close(); nextResponse.cancel(true); bigQueryReadClient.close(); } diff --git a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/ReadRowsHelper.java b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/ReadRowsHelper.java index 8f7b7991f97f..6c6408972d97 100644 --- a/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/ReadRowsHelper.java +++ b/plugin/trino-bigquery/src/main/java/io/trino/plugin/bigquery/ReadRowsHelper.java @@ -13,11 +13,13 @@ */ package io.trino.plugin.bigquery; +import com.google.api.gax.rpc.ServerStream; import com.google.cloud.bigquery.storage.v1.BigQueryReadClient; import com.google.cloud.bigquery.storage.v1.ReadRowsRequest; import com.google.cloud.bigquery.storage.v1.ReadRowsResponse; import io.airlift.log.Logger; +import java.io.Closeable; import java.util.Iterator; import java.util.NoSuchElementException; @@ -38,39 +40,40 @@ public ReadRowsHelper(BigQueryReadClient client, String streamName, int maxReadR this.maxReadRowsRetries = maxReadRowsRetries; } - public Iterator readRows() + ReadRowsIterator readRows() { - Iterator serverResponses = fetchResponses(0); - return new ReadRowsIterator(this, serverResponses); + return new ReadRowsIterator(this, fetchResponses(0)); } // In order to enable testing - protected Iterator fetchResponses(long offset) + protected ServerStream fetchResponses(long offset) { log.debug("Reading rows from %s offset %s", streamName, offset); return client.readRowsCallable() .call(ReadRowsRequest.newBuilder() .setReadStream(streamName) .setOffset(offset) - .build()) - .iterator(); + .build()); } // Ported from https://github.com/GoogleCloudDataproc/spark-bigquery-connector/pull/150 - private static class ReadRowsIterator - implements Iterator + static final class ReadRowsIterator + implements Iterator, Closeable { private final ReadRowsHelper helper; private long nextOffset; + private volatile ServerStream serverStream; private Iterator serverResponses; private int retries; + private volatile boolean closed; public ReadRowsIterator( ReadRowsHelper helper, - Iterator serverResponses) + ServerStream serverStream) { this.helper = helper; - this.serverResponses = serverResponses; + this.serverStream = serverStream; + this.serverResponses = serverStream.iterator(); } @Override @@ -93,7 +96,11 @@ public ReadRowsResponse next() // if relevant, retry the read, from the last read position if (BigQueryUtil.isRetryable(e) && retries < helper.maxReadRowsRetries) { log.debug("Request failed, retrying: %s", e); - serverResponses = helper.fetchResponses(nextOffset); + serverStream = helper.fetchResponses(nextOffset); + serverResponses = serverStream.iterator(); + if (closed) { + serverStream.cancel(); + } retries++; } else { @@ -106,5 +113,13 @@ public ReadRowsResponse next() throw new NoSuchElementException("No more server responses"); } + + @Override + public void close() + { + closed = true; + // Closing the client does not cancel in-flight calls, which keep holding read connection quota + serverStream.cancel(); + } } } diff --git a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSource.java b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSource.java index 55dc618d251b..ae962910f9a5 100644 --- a/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSource.java +++ b/plugin/trino-bigquery/src/test/java/io/trino/plugin/bigquery/TestBigQueryStoragePageSource.java @@ -30,6 +30,7 @@ import io.grpc.Server; import io.grpc.inprocess.InProcessChannelBuilder; import io.grpc.inprocess.InProcessServerBuilder; +import io.grpc.stub.ServerCallStreamObserver; import io.grpc.stub.StreamObserver; import io.trino.spi.PageBuilder; import io.trino.spi.TrinoException; @@ -64,13 +65,17 @@ import java.util.ArrayList; import java.util.List; import java.util.OptionalInt; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicInteger; import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; +import static io.airlift.concurrent.Threads.daemonThreadsNamed; import static io.trino.spi.StandardErrorCode.EXCEEDED_LOCAL_MEMORY_LIMIT; import static io.trino.spi.type.BigintType.BIGINT; import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER; +import static java.util.concurrent.Executors.newCachedThreadPool; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; @@ -90,12 +95,15 @@ final class TestBigQueryStoragePageSource // Fetch the responses synchronously, so that every page is available when requested private final ExecutorService executor = newDirectExecutorService(); + // Fetch the responses in the background, so that a fetch can wait for a response that never comes + private final ExecutorService backgroundExecutor = newCachedThreadPool(daemonThreadsNamed("test-bigquery-read-%s")); private final BigQueryArrowBufferAllocator arrowBufferAllocator = new BigQueryArrowBufferAllocator(new BigQueryArrowConfig(), new BigQueryArrowAllocatorStats()); @AfterAll void tearDown() { executor.shutdownNow(); + backgroundExecutor.shutdownNow(); } @Test @@ -104,7 +112,7 @@ void testArrowMemoryUsage() { try (TestingBigQueryReadServer server = new TestingBigQueryReadServer(ImmutableList.of(arrowResponse(0, 100), arrowResponse(100, 50)))) { List reportedBytes = new ArrayList<>(); - assertMemoryUsage(createArrowPageSource(server.client(), reportedBytes::add), reportedBytes); + assertMemoryUsage(createArrowPageSource(server.client(), executor, reportedBytes::add), reportedBytes); } } @@ -114,7 +122,7 @@ void testAvroMemoryUsage() { try (TestingBigQueryReadServer server = new TestingBigQueryReadServer(ImmutableList.of(avroResponse(0, 100), avroResponse(100, 50)))) { List reportedBytes = new ArrayList<>(); - assertMemoryUsage(createAvroPageSource(server.client(), reportedBytes::add), reportedBytes); + assertMemoryUsage(createAvroPageSource(server.client(), executor, reportedBytes::add), reportedBytes); } } @@ -124,7 +132,7 @@ void testArrowMemoryLimitExceededOnCreation() { try (TestingBigQueryReadServer server = new TestingBigQueryReadServer(ImmutableList.of())) { TrinoException memoryLimitExceeded = new TrinoException(EXCEEDED_LOCAL_MEMORY_LIMIT, "Exceeded memory limit"); - assertThatThrownBy(() -> createArrowPageSource(server.client(), _ -> { + assertThatThrownBy(() -> createArrowPageSource(server.client(), executor, _ -> { throw memoryLimitExceeded; })).isSameAs(memoryLimitExceeded); assertThat(server.readRowsRequests()).isZero(); @@ -137,13 +145,41 @@ void testAvroMemoryLimitExceededOnCreation() { try (TestingBigQueryReadServer server = new TestingBigQueryReadServer(ImmutableList.of())) { TrinoException memoryLimitExceeded = new TrinoException(EXCEEDED_LOCAL_MEMORY_LIMIT, "Exceeded memory limit"); - assertThatThrownBy(() -> createAvroPageSource(server.client(), _ -> { + assertThatThrownBy(() -> createAvroPageSource(server.client(), executor, _ -> { throw memoryLimitExceeded; })).isSameAs(memoryLimitExceeded); assertThat(server.readRowsRequests()).isZero(); } } + @Test + void testArrowCloseCancelsUnfinishedStream() + throws Exception + { + try (TestingBigQueryReadServer server = new TestingBigQueryReadServer(ImmutableList.of(arrowResponse(0, 100)), false)) { + assertCloseCancelsUnfinishedStream(createArrowPageSource(server.client(), backgroundExecutor, _ -> {}), server); + } + } + + @Test + void testAvroCloseCancelsUnfinishedStream() + throws Exception + { + try (TestingBigQueryReadServer server = new TestingBigQueryReadServer(ImmutableList.of(avroResponse(0, 100)), false)) { + assertCloseCancelsUnfinishedStream(createAvroPageSource(server.client(), backgroundExecutor, _ -> {}), server); + } + } + + private static void assertCloseCancelsUnfinishedStream(ConnectorPageSource pageSource, TestingBigQueryReadServer server) + throws Exception + { + pageSource.isBlocked().get(10, SECONDS); + assertPage(pageSource.getNextSourcePage(), 0, 100); + + pageSource.close(); + assertThat(server.awaitReadRowsCancelled()).isTrue(); + } + private static void assertMemoryUsage(ConnectorPageSource pageSource, List reportedBytes) throws IOException { @@ -173,13 +209,13 @@ private static void assertPage(SourcePage page, long firstValue, int positionCou } } - private ConnectorPageSource createArrowPageSource(BigQueryReadClient client, MemoryContext memoryContext) + private ConnectorPageSource createArrowPageSource(BigQueryReadClient client, ExecutorService executor, MemoryContext memoryContext) { BigQuerySplit split = BigQuerySplit.forStream("trace", "stream", ARROW_SCHEMA.toJson(), COLUMNS, OptionalInt.empty()); return new BigQueryStorageArrowPageSource(TYPE_MANAGER, client, executor, arrowBufferAllocator, 0, split, COLUMNS, memoryContext); } - private ConnectorPageSource createAvroPageSource(BigQueryReadClient client, MemoryContext memoryContext) + private ConnectorPageSource createAvroPageSource(BigQueryReadClient client, ExecutorService executor, MemoryContext memoryContext) { BigQuerySplit split = BigQuerySplit.forStream("trace", "stream", AVRO_SCHEMA.toString(), COLUMNS, OptionalInt.empty()); return new BigQueryStorageAvroPageSource(client, executor, TYPE_MANAGER, 0, split, COLUMNS, memoryContext); @@ -230,12 +266,19 @@ private static class TestingBigQueryReadServer implements AutoCloseable { private final AtomicInteger readRowsRequests = new AtomicInteger(); + private final CountDownLatch readRowsCancelled = new CountDownLatch(1); private final Server server; private final ManagedChannel channel; private final BigQueryReadClient client; public TestingBigQueryReadServer(List responses) throws IOException + { + this(responses, true); + } + + public TestingBigQueryReadServer(List responses, boolean completeStream) + throws IOException { String serverName = InProcessServerBuilder.generateName(); server = InProcessServerBuilder.forName(serverName) @@ -246,8 +289,12 @@ public TestingBigQueryReadServer(List responses) public void readRows(ReadRowsRequest request, StreamObserver responseObserver) { readRowsRequests.incrementAndGet(); + ((ServerCallStreamObserver) responseObserver).setOnCancelHandler(readRowsCancelled::countDown); responses.forEach(responseObserver::onNext); - responseObserver.onCompleted(); + // Without completing, the stream stays open like a BigQuery stream with rows left to read + if (completeStream) { + responseObserver.onCompleted(); + } } }) .build() @@ -271,6 +318,12 @@ public int readRowsRequests() return readRowsRequests.get(); } + public boolean awaitReadRowsCancelled() + throws InterruptedException + { + return readRowsCancelled.await(10, SECONDS); + } + @Override public void close() {